From fb7fce1a438c997ae379f7b5fd178d419b316d79 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 00:07:31 +0100
Subject: [PATCH 01/19] defuse

---
 cmd/zurg/main.go          |  40 ----
 go.mod                    |   2 -
 go.sum                    |   6 -
 internal/zfs/fs.go        |  35 ----
 internal/zfs/mount.go     |  57 ------
 internal/zfs/object.go    | 189 -----------------
 pkg/chunk/chunk.go        |  51 -----
 pkg/chunk/download.go     | 147 -------------
 pkg/chunk/manager.go      | 265 ------------------------
 pkg/chunk/manager_test.go |  53 -----
 pkg/chunk/stack.go        |  75 -------
 pkg/chunk/stack_test.go   |  72 -------
 pkg/chunk/storage.go      | 420 --------------------------------------
 13 files changed, 1412 deletions(-)
 delete mode 100644 internal/zfs/fs.go
 delete mode 100644 internal/zfs/mount.go
 delete mode 100644 internal/zfs/object.go
 delete mode 100644 pkg/chunk/chunk.go
 delete mode 100644 pkg/chunk/download.go
 delete mode 100644 pkg/chunk/manager.go
 delete mode 100644 pkg/chunk/manager_test.go
 delete mode 100644 pkg/chunk/stack.go
 delete mode 100644 pkg/chunk/stack_test.go
 delete mode 100644 pkg/chunk/storage.go

diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index b52100b..9399474 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -6,15 +6,12 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
-	"runtime"
 	"syscall"
 	"time"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/net"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
-	"github.com/debridmediamanager.com/zurg/internal/zfs"
-	"github.com/debridmediamanager.com/zurg/pkg/chunk"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/hashicorp/golang-lru/v2/expirable"
 )
@@ -38,27 +35,6 @@ func main() {
 	addr := fmt.Sprintf(":%s", config.GetPort())
 	server := &http.Server{Addr: addr, Handler: mux}
 
-	mountPoint := config.GetMountPoint()
-	if _, err := os.Stat(mountPoint); os.IsNotExist(err) {
-		if err := os.Mkdir(mountPoint, 0755); err != nil {
-			log.Panicf("Failed to create mount point: %v", err)
-		}
-	}
-
-	log.Debugf("Initializing chunk manager, cores: %d", runtime.NumCPU())
-	// 64kb request size
-	chunkMgr, err := chunk.NewManager(
-		"",
-		524288,                     // 512kb
-		1,                          // 1 chunk - load ahead (1MB total)
-		max(runtime.NumCPU()/2, 1), // check threads
-		max(runtime.NumCPU()/2, 1), // load threads
-		runtime.NumCPU()*2,         // max chunks
-		config)
-	if nil != err {
-		log.Panicf("Failed to initialize chunk manager: %v", err)
-	}
-
 	shutdown := make(chan os.Signal, 1)
 	signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
 
@@ -74,19 +50,6 @@ func main() {
 		}
 	}()
 
-	// Start the mount in a goroutine with panic recovery.
-	go func() {
-		defer func() {
-			if r := recover(); r != nil {
-				log.Errorf("Mount panic: %v\n", r)
-			}
-		}()
-		log.Infof("Mounting on %s", mountPoint)
-		if err := zfs.Mount(mountPoint, config, torrentMgr, chunkMgr); err != nil {
-			log.Panicf("Failed to mount: %v", err)
-		}
-	}()
-
 	<-shutdown
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -95,9 +58,6 @@ func main() {
 	if err := server.Shutdown(ctx); err != nil {
 		log.Errorf("Server shutdown error: %v\n", err)
 	}
-	if err := zfs.Unmount(mountPoint); err != nil {
-		log.Errorf("Unmount error: %v\n", err)
-	}
 
 	log.Info("BYE")
 }
diff --git a/go.mod b/go.mod
index eefa01b..69b6a67 100644
--- a/go.mod
+++ b/go.mod
@@ -3,10 +3,8 @@ module github.com/debridmediamanager.com/zurg
 go 1.21.3
 
 require (
-	bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	go.uber.org/zap v1.26.0
-	golang.org/x/sys v0.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
diff --git a/go.sum b/go.sum
index 252ca36..f49f369 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,3 @@
-bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 h1:A0NsYy4lDBZAC6QiYeJ4N+XuHIKBpyhAVRMHRQZKTeQ=
-bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5/go.mod h1:gG3RZAMXCa/OTes6rr9EwusmR1OH1tDDy+cg9c5YliY=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -8,16 +6,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
-github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
-golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/internal/zfs/fs.go b/internal/zfs/fs.go
deleted file mode 100644
index 4e6d74c..0000000
--- a/internal/zfs/fs.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package zfs
-
-import (
-	"os"
-	"sync"
-	"time"
-
-	"bazil.org/fuse/fs"
-	"github.com/debridmediamanager.com/zurg/internal/config"
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-	"github.com/debridmediamanager.com/zurg/pkg/chunk"
-	"go.uber.org/zap"
-)
-
-type FS struct {
-	uid      uint32
-	gid      uint32
-	umask    os.FileMode
-	directIO bool
-	lock     sync.RWMutex
-	c        config.ConfigInterface
-	t        *torrent.TorrentManager
-	log      *zap.SugaredLogger
-	initTime time.Time
-	chunk    *chunk.Manager
-}
-
-// Root returns the root path
-func (f *FS) Root() (fs.Node, error) {
-	return Object{
-		fs:      f,
-		objType: ROOT,
-		mtime:   f.initTime,
-	}, nil
-}
diff --git a/internal/zfs/mount.go b/internal/zfs/mount.go
deleted file mode 100644
index 7145a99..0000000
--- a/internal/zfs/mount.go
+++ /dev/null
@@ -1,57 +0,0 @@
-package zfs
-
-import (
-	"os"
-	"time"
-
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
-	"github.com/debridmediamanager.com/zurg/internal/config"
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-	"github.com/debridmediamanager.com/zurg/pkg/chunk"
-	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-
-	"golang.org/x/sys/unix"
-)
-
-func Mount(mountpoint string, cfg config.ConfigInterface, tMgr *torrent.TorrentManager, cMgr *chunk.Manager) error {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("zfs")
-
-	options := []fuse.MountOption{
-		fuse.AllowOther(),
-		fuse.AllowNonEmptyMount(),
-		fuse.MaxReadahead(uint32(128 << 10)),
-		fuse.DefaultPermissions(),
-		fuse.FSName("zurgfs"),
-	}
-	conn, err := fuse.Mount(mountpoint, options...)
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-
-	srv := fs.New(conn, nil)
-
-	filesys := &FS{
-		uid:      uint32(unix.Geteuid()),
-		gid:      uint32(unix.Getegid()),
-		umask:    os.FileMode(0),
-		c:        cfg,
-		t:        tMgr,
-		log:      log,
-		initTime: time.Now(),
-		chunk:    cMgr,
-	}
-
-	if err := srv.Serve(filesys); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func Unmount(mountpoint string) error {
-	fuse.Unmount(mountpoint)
-	return nil
-}
diff --git a/internal/zfs/object.go b/internal/zfs/object.go
deleted file mode 100644
index 8b4fe0e..0000000
--- a/internal/zfs/object.go
+++ /dev/null
@@ -1,189 +0,0 @@
-package zfs
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"syscall"
-	"time"
-
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// define variable as rootObject id
-const (
-	ROOT      = 0
-	DIRECTORY = 1
-	TORRENT   = 2
-	FILE      = 3
-)
-
-type Object struct {
-	fs         *FS
-	objType    int
-	parentName string
-	name       string
-	file       *torrent.File
-	size       uint64
-	mtime      time.Time
-}
-
-// Attr returns the attributes for a directory
-func (o Object) Attr(ctx context.Context, attr *fuse.Attr) error {
-	if o.objType == FILE {
-		attr.Mode = 0644
-	} else {
-		attr.Mode = os.ModeDir | 0755
-	}
-	attr.Size = o.size
-
-	attr.Uid = o.fs.uid
-	attr.Gid = o.fs.gid
-
-	attr.Ctime = o.mtime
-	attr.Mtime = o.mtime
-
-	attr.Blocks = (attr.Size + 511) >> 9
-
-	return nil
-}
-
-// ReadDirAll shows all files in the current directory
-func (o Object) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
-	dirs := []fuse.Dirent{}
-	switch o.objType {
-	case ROOT:
-		for _, directory := range o.fs.c.GetDirectories() {
-			dirs = append(dirs, fuse.Dirent{
-				Name: directory,
-				Type: fuse.DT_Dir,
-			})
-		}
-	case DIRECTORY:
-		seen := make(map[string]bool)
-		for _, item := range o.fs.t.GetByDirectory(o.name) {
-			if item.Progress != 100 {
-				continue
-			}
-			if _, exists := seen[item.Name]; exists {
-				continue
-			}
-			seen[item.Name] = true
-			dirs = append(dirs, fuse.Dirent{
-				Name: item.Name,
-				Type: fuse.DT_Dir,
-			})
-		}
-	case TORRENT:
-		finalName := make(map[string]bool)
-		for _, item := range o.fs.t.FindAllTorrentsWithName(o.parentName, o.name) {
-			for _, file := range item.SelectedFiles {
-				if file.Link == "" {
-					// log.Println("File has no link, skipping", file.Path)
-					continue
-				}
-				filename := filepath.Base(file.Path)
-				if finalName[filename] {
-					// fragment := davextra.GetLinkFragment(file.Link)
-					// filename = davextra.InsertLinkFragment(filename, fragment)
-					continue
-				}
-				finalName[filename] = true
-				dirs = append(dirs, fuse.Dirent{
-					Name: filename,
-					Type: fuse.DT_File,
-				})
-			}
-		}
-	}
-	return dirs, nil
-}
-
-// Lookup tests if a file is existent in the current directory
-func (o Object) Lookup(ctx context.Context, name string) (fs.Node, error) {
-	switch o.objType {
-	case ROOT:
-		for _, directory := range o.fs.c.GetDirectories() {
-			if directory == name {
-				return Object{
-					fs:         o.fs,
-					objType:    DIRECTORY,
-					parentName: o.name,
-					name:       name,
-					mtime:      o.fs.initTime,
-				}, nil
-			}
-		}
-	case DIRECTORY:
-		for _, item := range o.fs.t.GetByDirectory(o.name) {
-			if item.Name == name && item.Progress == 100 {
-				return Object{
-					fs:         o.fs,
-					objType:    TORRENT,
-					parentName: o.name,
-					name:       name,
-					mtime:      convertRFC3339toTime(item.Added),
-				}, nil
-			}
-		}
-	case TORRENT:
-		for _, item := range o.fs.t.FindAllTorrentsWithName(o.parentName, o.name) {
-			for _, file := range item.SelectedFiles {
-				if strings.HasSuffix(file.Path, name) && file.Link != "" {
-					return Object{
-						fs:         o.fs,
-						objType:    FILE,
-						parentName: o.name,
-						name:       name,
-						file:       &file,
-						size:       uint64(file.Bytes),
-						mtime:      convertRFC3339toTime(item.Added),
-					}, nil
-				}
-			}
-		}
-	}
-	return nil, syscall.ENOENT
-}
-
-// Open a file
-func (o Object) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
-	resp.Flags |= fuse.OpenDirectIO
-	return o, nil
-}
-
-// Read reads some bytes or the whole file
-func (o Object) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
-	o.fs.log.Debugf("Read %s (total size %d) req offset %d req size %d", o.name, o.size, req.Offset, req.Size)
-	data, err := o.fs.chunk.GetChunk(o.file, req.Offset, int64(req.Size))
-	if nil != err {
-		o.fs.log.Warnf("%v", err)
-		return syscall.EIO
-	}
-
-	resp.Data = data
-	return nil
-}
-
-// Remove deletes an element
-func (o Object) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
-	return fmt.Errorf("Remove not yet implemented")
-}
-
-// Rename renames an element
-func (o Object) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error {
-	return fmt.Errorf("Rename not yet implemented")
-}
-
-func convertRFC3339toTime(input string) time.Time {
-	layout := "2006-01-02T15:04:05.000Z"
-	t, err := time.Parse(layout, input)
-	if err != nil {
-		return time.Now()
-	}
-	return t
-}
diff --git a/pkg/chunk/chunk.go b/pkg/chunk/chunk.go
deleted file mode 100644
index bc0c964..0000000
--- a/pkg/chunk/chunk.go
+++ /dev/null
@@ -1,51 +0,0 @@
-package chunk
-
-import (
-	"container/list"
-	"hash/crc32"
-)
-
-// Chunk of memory
-type Chunk struct {
-	clean bool
-	*chunkHeader
-	bytes []byte
-	item  *list.Element
-}
-
-type chunkHeader struct {
-	id       RequestID
-	size     uint32
-	checksum uint32
-}
-
-func (c *Chunk) valid(id RequestID) bool {
-	if c.id != id {
-		return false
-	}
-	if !c.clean {
-		c.clean = c.checksum == c.calculateChecksum()
-	}
-	return c.clean
-}
-
-func (c *Chunk) update(id RequestID, bytes []byte) {
-	c.id = id
-	c.size = uint32(copy(c.bytes, bytes))
-	c.checksum = c.calculateChecksum()
-	c.clean = true
-}
-
-func (c *Chunk) calculateChecksum() uint32 {
-	size := c.size
-	if nil == c.bytes || size == 0 {
-		return 0
-	}
-	maxSize := uint32(len(c.bytes))
-	if size > maxSize {
-		// corrupt size or truncated chunk, fix size
-		c.size = maxSize
-		return crc32.Checksum(c.bytes, crc32Table)
-	}
-	return crc32.Checksum(c.bytes[:size], crc32Table)
-}
diff --git a/pkg/chunk/download.go b/pkg/chunk/download.go
deleted file mode 100644
index a9cd6ec..0000000
--- a/pkg/chunk/download.go
+++ /dev/null
@@ -1,147 +0,0 @@
-package chunk
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"sync"
-	"syscall"
-	"time"
-
-	"github.com/debridmediamanager.com/zurg/internal/config"
-	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
-	"go.uber.org/zap"
-	"golang.org/x/sys/unix"
-)
-
-// Downloader handles concurrent chunk downloads
-type Downloader struct {
-	BufferSize int64
-	queue      chan *Request
-	callbacks  map[RequestID][]DownloadCallback
-	lock       sync.Mutex
-	storage    *Storage
-	c          config.ConfigInterface
-	log        *zap.SugaredLogger
-}
-
-type DownloadCallback func(error, []byte)
-
-// NewDownloader creates a new download manager
-func NewDownloader(threads int, storage *Storage, bufferSize int64, c config.ConfigInterface) (*Downloader, error) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("downloader")
-
-	manager := Downloader{
-		BufferSize: bufferSize,
-		queue:      make(chan *Request, 100),
-		callbacks:  make(map[RequestID][]DownloadCallback, 100),
-		storage:    storage,
-		c:          c,
-		log:        log,
-	}
-
-	for i := 0; i < threads; i++ {
-		go manager.thread(i)
-	}
-
-	return &manager, nil
-}
-
-// Download starts a new download request
-func (d *Downloader) Download(req *Request, callback DownloadCallback) {
-	d.lock.Lock()
-	callbacks, exists := d.callbacks[req.id]
-	if nil != callback {
-		d.callbacks[req.id] = append(callbacks, callback)
-	} else if !exists {
-		d.callbacks[req.id] = callbacks
-	}
-	if !exists {
-		d.queue <- req
-	}
-	d.lock.Unlock()
-}
-
-func (d *Downloader) thread(n int) {
-	buffer, err := unix.Mmap(-1, 0, int(d.BufferSize), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE)
-	if nil != err {
-		d.log.Warnf("Failed to mmap download buffer %v: %v", n, err)
-		buffer = make([]byte, d.BufferSize)
-	}
-	for {
-		req := <-d.queue
-		d.download(req, buffer)
-	}
-}
-
-func (d *Downloader) download(req *Request, buffer []byte) {
-	d.log.Debugf("Starting download %v (preload: %v)", req.id, req.preload)
-	err := d.downloadFromAPI(req, buffer, 0)
-
-	d.lock.Lock()
-	callbacks := d.callbacks[req.id]
-	for _, callback := range callbacks {
-		callback(err, buffer)
-	}
-	delete(d.callbacks, req.id)
-	d.lock.Unlock()
-
-	if nil != err {
-		return
-	}
-
-	if err := d.storage.Store(req.id, buffer); nil != err {
-		d.log.Warnf("Could not store chunk %v: %v", req.id, err)
-	}
-}
-
-func (d *Downloader) downloadFromAPI(request *Request, buffer []byte, delay int64) error {
-	// sleep if request is throttled
-	if delay > 0 {
-		time.Sleep(time.Duration(delay) * time.Second)
-	}
-
-	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
-		return realdebrid.UnrestrictLink(d.c.GetToken(), request.file.Link)
-	}
-	resp := realdebrid.RetryUntilOk(unrestrictFn)
-	if resp == nil {
-		return fmt.Errorf("cannot unrestrict file %s %s", request.file.Path, request.file.Link)
-	}
-
-	downloadURL := resp.Download
-	req, err := http.NewRequest("GET", downloadURL, nil)
-	if nil != err {
-		d.log.Debugf("request init error: %v", err)
-		return fmt.Errorf("could not create request object %s %s from API", request.file.Path, request.file.Link)
-	}
-	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", request.offsetStart, request.offsetEnd-1))
-
-	res, err := http.DefaultClient.Do(req)
-	if nil != err {
-		d.log.Debugf("request error: %v", err)
-		return fmt.Errorf("could not request object %s %s from API", request.file.Path, request.file.Link)
-	}
-	defer res.Body.Close()
-	reader := res.Body
-
-	if res.StatusCode != 206 && res.StatusCode != 200 {
-		return fmt.Errorf("could not read object %s %s / StatusCode: %v",
-			request.file.Path, request.file.Link, res.StatusCode)
-	}
-
-	if res.ContentLength == -1 {
-		return fmt.Errorf("missing Content-Length header in response")
-	}
-
-	n, err := io.ReadFull(reader, buffer[:res.ContentLength:cap(buffer)])
-	if nil != err && err != io.ErrUnexpectedEOF {
-		d.log.Debugf("response read error: %v", err)
-		return fmt.Errorf("could not read objects %s %s API response", request.file.Path, request.file.Link)
-	}
-	d.log.Debugf("Downloaded %v bytes of %s %s", n, request.file.Path, request.file.Link)
-
-	return nil
-}
diff --git a/pkg/chunk/manager.go b/pkg/chunk/manager.go
deleted file mode 100644
index 6713c49..0000000
--- a/pkg/chunk/manager.go
+++ /dev/null
@@ -1,265 +0,0 @@
-package chunk
-
-import (
-	"crypto/sha256"
-	"encoding/binary"
-	"fmt"
-	"os"
-
-	"github.com/debridmediamanager.com/zurg/internal/config"
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// Manager manages chunks on disk
-type Manager struct {
-	ChunkSize  int64
-	LoadAhead  int
-	downloader *Downloader
-	storage    *Storage
-	queue      chan *QueueEntry
-}
-
-type QueueEntry struct {
-	request  *Request
-	response chan Response
-}
-
-// RequestID is the binary identifier for a chunk request
-type RequestID [24]byte
-
-func (id RequestID) String() string {
-	return fmt.Sprintf("%032x:%v", id[:16], binary.BigEndian.Uint64(id[16:]))
-}
-
-// Request represents a chunk request
-type Request struct {
-	id             RequestID
-	file           *torrent.File
-	offsetStart    int64
-	offsetEnd      int64
-	chunkOffset    int64
-	chunkOffsetEnd int64
-	sequence       int
-	preload        bool
-}
-
-// Response represetns a chunk response
-type Response struct {
-	Sequence int
-	Error    error
-	Bytes    []byte
-}
-
-// NewManager creates a new chunk manager
-func NewManager(
-	chunkFile string,
-	chunkSize int64,
-	loadAhead,
-	checkThreads,
-	loadThreads,
-	maxChunks int,
-	c config.ConfigInterface) (*Manager, error) {
-
-	pageSize := int64(os.Getpagesize())
-	if chunkSize < pageSize {
-		return nil, fmt.Errorf("chunk size must not be < %v", pageSize)
-	}
-	if chunkSize%pageSize != 0 {
-		return nil, fmt.Errorf("chunk size must be divideable by %v", pageSize)
-	}
-	// 32-Bit: ~2GB / 64-Bit: ~8EB
-	maxMmapSize := int64(^uint(0) >> 1)
-	if chunkSize > maxMmapSize {
-		return nil, fmt.Errorf("chunk size must be < %v", maxMmapSize)
-	}
-	if maxChunks < 2 || maxChunks < loadAhead {
-		return nil, fmt.Errorf("max-chunks must be greater than 2 and bigger than the load ahead value")
-	}
-
-	storage, err := NewStorage(chunkSize, maxChunks, maxMmapSize, chunkFile)
-	if nil != err {
-		return nil, err
-	}
-
-	downloader, err := NewDownloader(loadThreads, storage, chunkSize, c)
-	if nil != err {
-		return nil, err
-	}
-
-	manager := Manager{
-		ChunkSize:  chunkSize,
-		LoadAhead:  loadAhead,
-		downloader: downloader,
-		storage:    storage,
-		queue:      make(chan *QueueEntry, 100),
-	}
-
-	if err := manager.storage.Clear(); nil != err {
-		return nil, err
-	}
-
-	for i := 0; i < checkThreads; i++ {
-		go manager.thread()
-	}
-
-	return &manager, nil
-}
-
-// GetChunk loads one chunk and starts the preload for the next chunks
-func (m *Manager) GetChunk(object *torrent.File, offset, size int64) ([]byte, error) {
-	maxOffset := object.Bytes
-	if offset > maxOffset {
-		return nil, fmt.Errorf("tried to read past EOF of %v at offset %v", object.ID, offset)
-	}
-	// Log.Infof("Request %v:%v md5:%v", object.ID, offset, object.MD5Checksum)
-	if offset+size > maxOffset {
-		size = object.Bytes - offset
-	}
-
-	ranges := splitChunkRanges(offset, size, m.ChunkSize)
-	numRanges := len(ranges)
-	responses := make(chan Response, numRanges)
-
-	last := numRanges - 1
-	for i, r := range ranges {
-		m.requestChunk(object, r.offset, r.size, i, i == last, responses)
-	}
-
-	data := make([]byte, size)
-	for i := 0; i < cap(responses); i++ {
-		res := <-responses
-		if nil != res.Error {
-			return nil, res.Error
-		}
-
-		dataOffset := ranges[res.Sequence].offset - offset
-
-		if n := copy(data[dataOffset:], res.Bytes); n == 0 {
-			return nil, fmt.Errorf("request %v slice %v has empty response", object.ID, res.Sequence)
-		}
-	}
-	close(responses)
-
-	return data, nil
-}
-
-func buildRequestID(object *torrent.File, offset int64) (id RequestID) {
-	fileID := object.Link
-	if fileID == "" {
-		fileID = object.Path
-	}
-	hash := sha256.Sum256([]byte(fileID))
-	copy(id[:16], hash[:16])
-	binary.BigEndian.PutUint64(id[16:], uint64(offset))
-	return
-}
-
-func (m *Manager) requestChunk(object *torrent.File, offset, size int64, sequence int, preload bool, response chan Response) {
-	chunkOffset := offset % m.ChunkSize
-	offsetStart := offset - chunkOffset
-	offsetEnd := offsetStart + m.ChunkSize
-
-	request := &Request{
-		id:             buildRequestID(object, offsetStart),
-		file:           object,
-		offsetStart:    offsetStart,
-		offsetEnd:      offsetEnd,
-		chunkOffset:    chunkOffset,
-		chunkOffsetEnd: chunkOffset + size,
-		sequence:       sequence,
-		preload:        false,
-	}
-
-	m.queue <- &QueueEntry{
-		request:  request,
-		response: response,
-	}
-
-	if !preload {
-		return
-	}
-
-	for i := m.ChunkSize; i < (m.ChunkSize * int64(m.LoadAhead+1)); i += m.ChunkSize {
-		aheadOffsetStart := offsetStart + i
-		aheadOffsetEnd := aheadOffsetStart + m.ChunkSize
-		if uint64(aheadOffsetStart) < uint64(object.Bytes) && uint64(aheadOffsetEnd) < uint64(object.Bytes) {
-			request := &Request{
-				id:          buildRequestID(object, aheadOffsetStart),
-				file:        object,
-				offsetStart: aheadOffsetStart,
-				offsetEnd:   aheadOffsetEnd,
-				preload:     true,
-			}
-			m.queue <- &QueueEntry{
-				request: request,
-			}
-		}
-	}
-}
-
-type byteRange struct {
-	offset, size int64
-}
-
-// Calculate request ranges that span multiple chunks
-//
-// This can happen with Direct-IO and unaligned reads or
-// if the size is bigger than the chunk size.
-func splitChunkRanges(offset, size, chunkSize int64) []byteRange {
-	ranges := make([]byteRange, 0, size/chunkSize+2)
-	for remaining := size; remaining > 0; remaining -= size {
-		size = min(remaining, chunkSize-offset%chunkSize)
-		ranges = append(ranges, byteRange{offset, size})
-		offset += size
-	}
-	return ranges
-}
-
-func (m *Manager) thread() {
-	for {
-		queueEntry := <-m.queue
-		m.checkChunk(queueEntry.request, queueEntry.response)
-	}
-}
-
-func (m *Manager) checkChunk(req *Request, response chan Response) {
-	if nil == response {
-		if nil == m.storage.Load(req.id) {
-			m.downloader.Download(req, nil)
-		}
-		return
-	}
-
-	if bytes := m.storage.Load(req.id); nil != bytes {
-		response <- Response{
-			Sequence: req.sequence,
-			Bytes:    adjustResponseChunk(req, bytes),
-		}
-		return
-	}
-
-	m.downloader.Download(req, func(err error, bytes []byte) {
-		response <- Response{
-			Sequence: req.sequence,
-			Error:    err,
-			Bytes:    adjustResponseChunk(req, bytes),
-		}
-	})
-}
-
-func adjustResponseChunk(req *Request, bytes []byte) []byte {
-	if nil == bytes {
-		return nil
-	}
-	bytesLen := int64(len(bytes))
-	sOffset := min(req.chunkOffset, bytesLen)
-	eOffset := min(req.chunkOffsetEnd, bytesLen)
-	return bytes[sOffset:eOffset]
-}
-
-func min(x, y int64) int64 {
-	if x < y {
-		return x
-	}
-	return y
-}
diff --git a/pkg/chunk/manager_test.go b/pkg/chunk/manager_test.go
deleted file mode 100644
index ceb78e5..0000000
--- a/pkg/chunk/manager_test.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package chunk
-
-import "testing"
-
-func TestSplitChunkRanges(t *testing.T) {
-	testcases := []struct {
-		offset, size, chunkSize int64
-		result                  []byteRange
-	}{
-		{0, 0, 4096, []byteRange{}},
-		{0, 4096, 4096, []byteRange{
-			{0, 4096},
-		}},
-		{4095, 4096, 4096, []byteRange{
-			{4095, 1},
-			{4096, 4095},
-		}},
-		{0, 8192, 4096, []byteRange{
-			{0, 4096},
-			{4096, 4096},
-		}},
-		{2048, 8192, 4096, []byteRange{
-			{2048, 2048},
-			{4096, 4096},
-			{8192, 2048},
-		}},
-		{2048, 8192, 4096, []byteRange{
-			{2048, 2048},
-			{4096, 4096},
-			{8192, 2048},
-		}},
-		{17960960, 16777216, 10485760, []byteRange{
-			{17960960, 3010560},
-			{20971520, 10485760},
-			{31457280, 3280896},
-		}},
-	}
-	for i, tc := range testcases {
-		ranges := splitChunkRanges(tc.offset, tc.size, tc.chunkSize)
-		actualSize := len(ranges)
-		expectedSize := len(tc.result)
-		if actualSize != expectedSize {
-			t.Fatalf("ByteRange %v length mismatch: %v != %v", i, actualSize, expectedSize)
-		}
-		for j, r := range ranges {
-			actual := r
-			expected := tc.result[j]
-			if actual != expected {
-				t.Fatalf("ByteRange %v mismatch: %v != %v", i, actual, expected)
-			}
-		}
-	}
-}
diff --git a/pkg/chunk/stack.go b/pkg/chunk/stack.go
deleted file mode 100644
index 8955306..0000000
--- a/pkg/chunk/stack.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package chunk
-
-import (
-	"container/list"
-	"sync"
-)
-
-// Stack is a thread safe list/stack implementation
-type Stack struct {
-	items   *list.List
-	lock    sync.Mutex
-	maxSize int
-}
-
-// NewStack creates a new stack
-func NewStack(maxChunks int) *Stack {
-	return &Stack{
-		items:   list.New(),
-		maxSize: maxChunks,
-	}
-}
-
-// Len gets the number of items on the stack
-func (s *Stack) Len() int {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	return s.items.Len()
-}
-
-// Pop pops the first item from the stack
-func (s *Stack) Pop() int {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	if s.items.Len() < s.maxSize {
-		return -1
-	}
-	item := s.items.Front()
-	if nil == item {
-		return -1
-	}
-	s.items.Remove(item)
-	return item.Value.(int)
-}
-
-// Touch moves the specified item to the last position of the stack
-func (s *Stack) Touch(item *list.Element) {
-	s.lock.Lock()
-	if item != s.items.Back() {
-		s.items.MoveToBack(item)
-	}
-	s.lock.Unlock()
-}
-
-// Push adds a new item to the last position of the stack
-func (s *Stack) Push(id int) *list.Element {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	return s.items.PushBack(id)
-}
-
-// Prepend adds a list to the front of the stack
-func (s *Stack) Prepend(items *list.List) {
-	s.lock.Lock()
-	s.items.PushFrontList(items)
-	s.lock.Unlock()
-}
-
-// Purge an item from the stack
-func (s *Stack) Purge(item *list.Element) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	if item != s.items.Front() {
-		s.items.MoveToFront(item)
-	}
-}
diff --git a/pkg/chunk/stack_test.go b/pkg/chunk/stack_test.go
deleted file mode 100644
index 023bd70..0000000
--- a/pkg/chunk/stack_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-package chunk
-
-import "testing"
-
-func TestOOB(t *testing.T) {
-	stack := NewStack(1)
-
-	item := stack.Push(1)
-	stack.Touch(item)
-}
-
-func TestAddToStack(t *testing.T) {
-	stack := NewStack(1)
-
-	item1 := stack.Push(1)
-	item2 := stack.Push(2)
-	item3 := stack.Push(3)
-	item4 := stack.Push(4)
-
-	stack.Touch(item1)
-	stack.Touch(item3)
-
-	stack.Purge(item2)
-	stack.Purge(item4)
-
-	v := stack.Pop()
-	if v != 4 {
-		t.Fatalf("Expected 4 got %v", v)
-	}
-
-	v = stack.Pop()
-	if v != 2 {
-		t.Fatalf("Expected 2 got %v", v)
-	}
-
-	v = stack.Pop()
-	if v != 1 {
-		t.Fatalf("Expected 1 got %v", v)
-	}
-
-	v = stack.Pop()
-	if v != 3 {
-		t.Fatalf("Expected 3 got %v", v)
-	}
-
-	v = stack.Pop()
-	if v != -1 {
-		t.Fatalf("Expected -1 got %v", v)
-	}
-}
-
-func TestLen(t *testing.T) {
-	stack := NewStack(1)
-
-	v := stack.Len()
-	if v != 0 {
-		t.Fatalf("Expected 0 got %v", v)
-	}
-
-	stack.Push(1)
-	v = stack.Len()
-	if v != 1 {
-		t.Fatalf("Expected 1 got %v", v)
-	}
-
-	_ = stack.Pop()
-	v = stack.Len()
-	if v != 0 {
-		t.Fatalf("Expected 0 got %v", v)
-	}
-
-}
diff --git a/pkg/chunk/storage.go b/pkg/chunk/storage.go
deleted file mode 100644
index 264dcc8..0000000
--- a/pkg/chunk/storage.go
+++ /dev/null
@@ -1,420 +0,0 @@
-package chunk
-
-import (
-	"container/list"
-	"fmt"
-	"hash/crc32"
-	"os"
-	"os/signal"
-	"sync"
-	"syscall"
-	"time"
-	"unsafe"
-
-	"go.uber.org/zap"
-	"golang.org/x/sys/unix"
-
-	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-)
-
-const (
-	headerSize     = int(unsafe.Sizeof(*new(chunkHeader)))
-	tocSize        = int64(unsafe.Sizeof(*new(journalHeader)))
-	journalMagic   = uint16('P'<<8 | 'D'&0xFF)
-	journalVersion = uint8(2)
-)
-
-var (
-	blankRequestID RequestID
-	crc32Table     = crc32.MakeTable(crc32.Castagnoli)
-)
-
-// Storage is a chunk storage
-type Storage struct {
-	ChunkFile       *os.File
-	ChunkSize       int64
-	HeaderSize      int64
-	MaxChunks       int
-	chunks          map[RequestID]int
-	stack           *Stack
-	lock            sync.RWMutex
-	buffers         []*Chunk
-	loadChunks      int
-	signals         chan os.Signal
-	journal         []byte
-	mmapRegions     [][]byte
-	chunksPerRegion int64
-	log             *zap.SugaredLogger
-}
-
-type journalHeader struct {
-	magic      uint16
-	version    uint8
-	headerSize uint8
-	maxChunks  uint32
-	chunkSize  uint32
-	checksum   uint32
-}
-
-// NewStorage creates a new storage
-func NewStorage(chunkSize int64, maxChunks int, maxMmapSize int64, chunkFilePath string) (*Storage, error) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("storage")
-
-	s := Storage{
-		ChunkSize: chunkSize,
-		MaxChunks: maxChunks,
-		chunks:    make(map[RequestID]int, maxChunks),
-		stack:     NewStack(maxChunks),
-		buffers:   make([]*Chunk, maxChunks),
-		signals:   make(chan os.Signal, 1),
-		log:       log,
-	}
-
-	journalSize := tocSize + int64(headerSize*maxChunks)
-	journalOffset := chunkSize * int64(maxChunks)
-
-	// Non-empty string in chunkFilePath enables MMAP disk storage for chunks
-	if chunkFilePath != "" {
-		chunkFile, err := os.OpenFile(chunkFilePath, os.O_RDWR|os.O_CREATE, 0600)
-		if nil != err {
-			s.log.Debugf("%v", err)
-			return nil, fmt.Errorf("could not open chunk cache file")
-		}
-		s.ChunkFile = chunkFile
-		currentSize, err := chunkFile.Seek(0, os.SEEK_END)
-		if nil != err {
-			s.log.Debugf("%v", err)
-			return nil, fmt.Errorf("chunk file is not seekable")
-		}
-		wantedSize := journalOffset + journalSize
-		s.log.Debugf("Current chunk cache file size: %v B (wanted: %v B)", currentSize, wantedSize)
-		if err := chunkFile.Truncate(currentSize); nil != err {
-			s.log.Warnf("Could not truncate chunk cache, skip resizing")
-		} else if currentSize != wantedSize {
-			if currentSize > tocSize {
-				err = s.relocateJournal(currentSize, wantedSize, journalSize, journalOffset)
-				if nil != err {
-					s.log.Errorf("%v", err)
-				} else {
-					s.log.Infof("Relocated chunk cache journal")
-				}
-			}
-			if err := chunkFile.Truncate(wantedSize); nil != err {
-				s.log.Debugf("%v", err)
-				return nil, fmt.Errorf("could not resize chunk cache file")
-			}
-		}
-		s.log.Infof("Created chunk cache file %v", chunkFile.Name())
-		s.loadChunks = int(min(currentSize/chunkSize, int64(maxChunks)))
-	}
-
-	// Alocate journal
-	if journal, err := s.mmap(journalOffset, journalSize); nil != err {
-		return nil, fmt.Errorf("could not allocate journal: %v", err)
-	} else {
-		if err := unix.Madvise(journal, syscall.MADV_RANDOM); nil != err {
-			s.log.Warnf("Madvise MADV_RANDOM for journal failed: %v", err)
-		}
-		tocOffset := journalSize - tocSize
-		header := journal[tocOffset:]
-		if valid := s.checkJournal(header, false); !valid {
-			s.initJournal(header)
-		}
-		s.journal = journal[:tocOffset]
-	}
-
-	// Setup sighandler
-	signal.Notify(s.signals, syscall.SIGINT, syscall.SIGTERM)
-
-	// Allocate mmap regions for chunks
-	if err := s.allocateMmapRegions(maxMmapSize); nil != err {
-		return nil, err
-	}
-	// Map chunks to slices from mmap regions
-	if err := s.mmapChunks(); nil != err {
-		return nil, err
-	}
-
-	return &s, nil
-}
-
-// relocateJournal moves existing journal prior to resize
-func (s *Storage) relocateJournal(currentSize, wantedSize, journalSize, journalOffset int64) error {
-	header := make([]byte, tocSize)
-	if _, err := s.ChunkFile.ReadAt(header, currentSize-tocSize); nil != err {
-		return fmt.Errorf("failed to read journal header: %v", err)
-	}
-
-	if valid := s.checkJournal(header, true); !valid {
-		return fmt.Errorf("failed to validate journal header")
-	}
-
-	h := (*journalHeader)(unsafe.Pointer(&header[0]))
-	oldJournalOffset := s.ChunkSize * int64(h.maxChunks)
-	oldJournalSize := min(journalSize, currentSize-oldJournalOffset) - tocSize
-	journal := make([]byte, journalSize)
-
-	if _, err := s.ChunkFile.ReadAt(journal[:oldJournalSize], oldJournalOffset); nil != err {
-		return fmt.Errorf("failed to read journal: %v", err)
-	}
-
-	s.initJournal(header)
-
-	sizeWithoutJournal := currentSize - oldJournalSize - tocSize
-	if err := s.ChunkFile.Truncate(sizeWithoutJournal); nil != err {
-		return fmt.Errorf("could not truncate chunk cache journal: %v", err)
-	}
-
-	if err := s.ChunkFile.Truncate(wantedSize); nil != err {
-		return fmt.Errorf("could not resize chunk cache file: %v", err)
-	}
-
-	if _, err := s.ChunkFile.WriteAt(journal, journalOffset); nil != err {
-		return fmt.Errorf("failed to write journal: %v", err)
-	}
-	if _, err := s.ChunkFile.WriteAt(header, wantedSize-tocSize); nil != err {
-		return fmt.Errorf("failed to write journal header: %v", err)
-	}
-	return nil
-}
-
-// checkJournal verifies the journal header
-func (s *Storage) checkJournal(journal []byte, skipMaxChunks bool) bool {
-	h := (*journalHeader)(unsafe.Pointer(&journal[0]))
-	// check magic bytes / endianess mismatch ('PD' vs 'DP')
-	if h.magic != journalMagic {
-		s.log.Debugf("Journal magic mismatch: %v != %v", h.magic, journalMagic)
-		return false
-	}
-	checksum := crc32.Checksum(journal[:12], crc32Table)
-	if h.checksum != checksum {
-		s.log.Debugf("Journal checksum mismatch: %08X != %08X", h.checksum, checksum)
-		return false
-	}
-	if h.version != journalVersion {
-		s.log.Debugf("Journal version mismatch: %v != %v", h.version, journalVersion)
-		return false
-	}
-	if h.headerSize != uint8(headerSize) {
-		s.log.Debugf("Journal chunk header size mismatch: %v != %v", h.headerSize, headerSize)
-		return false
-	}
-	if !skipMaxChunks && h.maxChunks != uint32(s.MaxChunks) {
-		s.log.Debugf("Journal max chunks mismatch: %v != %v", h.maxChunks, s.MaxChunks)
-		return false
-	}
-	if h.chunkSize != uint32(s.ChunkSize) {
-		s.log.Debugf("Journal chunk size mismatch: %v != %v", h.chunkSize, s.ChunkSize)
-		return false
-	}
-	s.log.Debug("Journal is valid")
-	return true
-}
-
-// initJournal initializes the journal
-func (s *Storage) initJournal(journal []byte) {
-	h := (*journalHeader)(unsafe.Pointer(&journal[0]))
-	h.magic = journalMagic
-	h.version = journalVersion
-	h.headerSize = uint8(headerSize)
-	h.maxChunks = uint32(s.MaxChunks)
-	h.chunkSize = uint32(s.ChunkSize)
-	h.checksum = crc32.Checksum(journal[:12], crc32Table)
-}
-
-// allocateMmapRegions creates memory mappings to fit all chunks
-func (s *Storage) allocateMmapRegions(maxMmapSize int64) error {
-	s.chunksPerRegion = maxMmapSize / s.ChunkSize
-	regionSize := s.chunksPerRegion * s.ChunkSize
-	numRegions := int64(s.MaxChunks) / s.chunksPerRegion
-	remChunks := int64(s.MaxChunks) % s.chunksPerRegion
-	if remChunks != 0 {
-		numRegions++
-	}
-	s.mmapRegions = make([][]byte, numRegions)
-	for i := int64(0); i < int64(len(s.mmapRegions)); i++ {
-		size := regionSize
-		if i == numRegions-1 && remChunks != 0 {
-			size = remChunks * s.ChunkSize
-		}
-		s.log.Debugf("Allocate mmap region %v/%v with size %v B", i+1, numRegions, size)
-		region, err := s.mmap(i*regionSize, size)
-		if nil != err {
-			s.log.Errorf("failed to mmap region %v/%v with size %v B", i+1, numRegions, size)
-			return err
-		}
-		if err := unix.Madvise(region, syscall.MADV_SEQUENTIAL); nil != err {
-			s.log.Warnf("Madvise MADV_SEQUENTIAL for region %v/%v failed: %v", i+1, numRegions, err)
-		}
-		s.mmapRegions[i] = region
-	}
-	return nil
-}
-
-// mmapChunks slices buffers from mmap regions and loads chunk metadata
-func (s *Storage) mmapChunks() error {
-	start := time.Now()
-	empty := list.New()
-	restored := list.New()
-	loadedChunks := 0
-	for i := 0; i < s.MaxChunks; i++ {
-		select {
-		case sig := <-s.signals:
-			s.log.Warnf("Received signal %v, aborting chunk loader", sig)
-			return fmt.Errorf("aborted by signal")
-		default:
-			if loaded, err := s.initChunk(i, empty, restored); nil != err {
-				s.log.Errorf("failed to allocate chunk %v: %v", i, err)
-				return fmt.Errorf("failed to initialize chunks")
-			} else if loaded {
-				loadedChunks++
-			}
-		}
-	}
-	s.stack.Prepend(restored)
-	s.stack.Prepend(empty)
-	elapsed := time.Since(start)
-	if nil != s.ChunkFile {
-		s.log.Infof("Loaded %v/%v cache chunks in %v", loadedChunks, s.MaxChunks, elapsed)
-	} else {
-		s.log.Infof("Allocated %v cache chunks in %v", s.MaxChunks, elapsed)
-	}
-	return nil
-}
-
-// initChunk tries to restore a chunk from disk
-func (s *Storage) initChunk(index int, empty *list.List, restored *list.List) (bool, error) {
-	chunk, err := s.allocateChunk(index)
-	if err != nil {
-		s.log.Debugf("%v", err)
-		return false, err
-	}
-
-	s.buffers[index] = chunk
-
-	id := chunk.id
-
-	if blankRequestID == id || index >= s.loadChunks {
-		chunk.item = empty.PushBack(index)
-		// s.log.Tracef("Allocate chunk %v/%v", index+1, s.MaxChunks)
-		return false, nil
-	}
-
-	chunk.item = restored.PushBack(index)
-	// s.log.Tracef("Load chunk %v/%v (restored: %v)", index+1, s.MaxChunks, id)
-	s.chunks[id] = index
-
-	return true, nil
-}
-
-// allocateChunk creates a new mmap-backed chunk
-func (s *Storage) allocateChunk(index int) (*Chunk, error) {
-	region := int64(index) / s.chunksPerRegion
-	offset := (int64(index) - region*s.chunksPerRegion) * s.ChunkSize
-	// s.log.Tracef("Allocate chunk %v from region %v at offset %v", index+1, region, offset)
-	bytes := s.mmapRegions[region][offset : offset+s.ChunkSize : offset+s.ChunkSize]
-	headerOffset := index * headerSize
-	header := (*chunkHeader)(unsafe.Pointer(&s.journal[headerOffset]))
-	chunk := Chunk{
-		chunkHeader: header,
-		bytes:       bytes,
-	}
-	return &chunk, nil
-}
-
-func (s *Storage) mmap(offset, size int64) ([]byte, error) {
-	if s.ChunkFile != nil {
-		return unix.Mmap(int(s.ChunkFile.Fd()), offset, int(size), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
-	} else {
-		return unix.Mmap(-1, 0, int(size), syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_ANON|syscall.MAP_PRIVATE)
-	}
-}
-
-// Clear removes all old chunks on disk (will be called on each program start)
-func (s *Storage) Clear() error {
-	return nil
-}
-
-// Load a chunk from ram or creates it
-func (s *Storage) Load(id RequestID) []byte {
-	s.lock.RLock()
-	chunk := s.fetch(id)
-	if nil == chunk {
-		// s.log.Tracef("Load chunk %v (missing)", id)
-		s.lock.RUnlock()
-		return nil
-	}
-	if chunk.clean {
-		// s.log.Tracef("Load chunk %v (clean)", id)
-		defer s.lock.RUnlock()
-		return chunk.bytes
-	}
-	s.lock.RUnlock()
-	// Switch to write lock to avoid races on crc verification
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	if chunk.valid(id) {
-		s.log.Debugf("Load chunk %v (verified)", id)
-		return chunk.bytes
-	}
-	s.log.Warnf("Load chunk %v (bad checksum: %08x <> %08x)", id, chunk.checksum, chunk.calculateChecksum())
-	s.stack.Purge(chunk.item)
-	return nil
-}
-
-// Store stores a chunk in the RAM and adds it to the disk storage queue
-func (s *Storage) Store(id RequestID, bytes []byte) (err error) {
-	s.lock.RLock()
-
-	// Avoid storing same chunk multiple times
-	chunk := s.fetch(id)
-	if nil != chunk && chunk.clean {
-		// s.log.Tracef("Create chunk %v (exists: clean)", id)
-		s.lock.RUnlock()
-		return nil
-	}
-
-	s.lock.RUnlock()
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	if nil != chunk {
-		if chunk.valid(id) {
-			s.log.Debugf("Create chunk %v (exists: valid)", id)
-			return nil
-		}
-		s.log.Warnf("Create chunk %v(exists: overwrite)", id)
-	} else {
-		index := s.stack.Pop()
-		if index == -1 {
-			s.log.Debugf("Create chunk %v (failed)", id)
-			return fmt.Errorf("no buffers available")
-		}
-		chunk = s.buffers[index]
-		deleteID := chunk.id
-		if blankRequestID != deleteID {
-			delete(s.chunks, deleteID)
-			s.log.Debugf("Create chunk %v (reused)", id)
-		} else {
-			s.log.Debugf("Create chunk %v (stored)", id)
-		}
-		s.chunks[id] = index
-		chunk.item = s.stack.Push(index)
-	}
-
-	chunk.update(id, bytes)
-
-	return nil
-}
-
-// fetch chunk and index by id
-func (s *Storage) fetch(id RequestID) *Chunk {
-	index, exists := s.chunks[id]
-	if !exists {
-		return nil
-	}
-	chunk := s.buffers[index]
-	s.stack.Touch(chunk.item)
-	return chunk
-}

From c895d57c89624188471aa66a1cd6bffd2a02a1d5 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 00:08:13 +0100
Subject: [PATCH 02/19] Publish to zurg-testing repo

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 946ecf6..e792dfc 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -10,7 +10,7 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository }}-testing
 
 jobs:
   build-and-push-image:

From e2a2d2caa6acdaa3506cfd05a3ebab38564aaecf Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 01:17:23 +0100
Subject: [PATCH 03/19] Fix error videos

---
 internal/dav/listing.go   |  8 ++------
 internal/http/listing.go  |  4 ++--
 internal/http/util.go     |  8 --------
 internal/universal/get.go | 37 ++++++++++++++++++++++++++-----------
 pkg/realdebrid/types.go   | 11 ++++++-----
 5 files changed, 36 insertions(+), 32 deletions(-)

diff --git a/internal/dav/listing.go b/internal/dav/listing.go
index 3d18ef8..d2653e6 100644
--- a/internal/dav/listing.go
+++ b/internal/dav/listing.go
@@ -42,13 +42,13 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 		output, err = handleSingleTorrent(requestPath, w, r, t)
 	default:
 		log.Errorf("Request %s %s not found", r.Method, requestPath)
-		writeHTTPError(w, "Not Found", http.StatusNotFound)
+		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
 
 	if err != nil {
 		log.Errorf("Error processing request: %v", err)
-		writeHTTPError(w, "Server error", http.StatusInternalServerError)
+		http.Error(w, "Server error", http.StatusInternalServerError)
 		return
 	}
 
@@ -107,7 +107,3 @@ func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Requ
 	}
 	return xml.Marshal(resp)
 }
-
-func writeHTTPError(w http.ResponseWriter, errorMessage string, statusCode int) {
-	http.Error(w, errorMessage, statusCode)
-}
diff --git a/internal/http/listing.go b/internal/http/listing.go
index cdd23f2..f641062 100644
--- a/internal/http/listing.go
+++ b/internal/http/listing.go
@@ -39,13 +39,13 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
 		output, err = handleSingleTorrent(requestPath, w, r, t)
 	default:
 		log.Errorf("Request %s %s not found", r.Method, requestPath)
-		writeHTTPError(w, "Not Found", http.StatusNotFound)
+		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
 
 	if err != nil {
 		log.Errorf("Error processing request: %v", err)
-		writeHTTPError(w, "Server error", http.StatusInternalServerError)
+		http.Error(w, "Server error", http.StatusInternalServerError)
 		return
 	}
 
diff --git a/internal/http/util.go b/internal/http/util.go
index 8e71035..e37f3b1 100644
--- a/internal/http/util.go
+++ b/internal/http/util.go
@@ -1,13 +1,5 @@
 package http
 
-import (
-	"net/http"
-)
-
-func writeHTTPError(w http.ResponseWriter, errorMessage string, statusCode int) {
-	http.Error(w, errorMessage, statusCode)
-}
-
 func removeEmptySegments(urlSegments []string) []string {
 	var result []string
 	for _, s := range urlSegments {
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 775d113..736f09b 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -43,7 +43,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		return
 	}
 	if data, exists := cache.Get(requestPath); exists {
-		streamFileToResponse(data, w, r, c.GetNetworkBufferSize(), log)
+		streamFileToResponse(data, w, r, c, log)
 		return
 	}
 
@@ -54,7 +54,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
 	if torrents == nil {
 		log.Errorf("Cannot find torrent %s in the directory %s", requestPath, baseDirectory)
-		http.Redirect(w, r, "https://send.nukes.wtf/tDeTd0", http.StatusFound)
+		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 
@@ -62,12 +62,13 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	torrent, file := getFile(torrents, filenameV2, linkFragment)
 	if file == nil {
 		log.Errorf("Cannot find file from path %s", requestPath)
-		http.Error(w, "Cannot find file", http.StatusNotFound)
+		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 	if file.Link == "" {
 		// This is a dead file, serve an alternate file
 		log.Errorf("File %s is no longer available", filename)
+		streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, c, log)
 		return
 	}
 	link := file.Link
@@ -81,26 +82,28 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 			log.Errorf("Cannot unrestrict file %s %s", filenameV2, link)
 			t.HideTheFile(torrent, file)
 		}
-		http.Redirect(w, r, "https://send.nukes.wtf/tDeTd0", http.StatusFound)
+		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, c, log)
 		return
 	} else if resp.Filename != filenameV2 {
 		actualExt := filepath.Ext(resp.Filename)
 		expectedExt := filepath.Ext(filenameV2)
-		if actualExt != expectedExt {
+		if actualExt != expectedExt && resp.Streamable != 1 {
 			log.Errorf("File extension mismatch: %s and %s", filenameV2, resp.Filename)
+			streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, c, log)
+			return
 		} else {
 			log.Errorf("Filename mismatch: %s and %s", filenameV2, resp.Filename)
 		}
 	}
 	cache.Add(requestPath, resp.Download)
-	streamFileToResponse(resp.Download, w, r, c.GetNetworkBufferSize(), log)
+	streamFileToResponse(resp.Download, w, r, c, log)
 }
 
-func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, bufferSize int, log *zap.SugaredLogger) {
+func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		log.Errorf("Error creating new request %v", err)
-		http.Redirect(w, r, "https://send.nukes.wtf/ARjVWb", http.StatusFound)
+		streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, c, log)
 		return
 	}
 
@@ -113,14 +116,14 @@ func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, bu
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		log.Errorf("Error downloading file %v", err)
-		http.Redirect(w, r, "https://send.nukes.wtf/TB2u2n", http.StatusFound)
+		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, c, log)
 		return
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
 		log.Errorf("Received a nonOK status code %d", resp.StatusCode)
-		http.Redirect(w, r, "https://send.nukes.wtf/b5AiON", http.StatusFound)
+		streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, c, log)
 		return
 	}
 
@@ -130,6 +133,18 @@ func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, bu
 		}
 	}
 
-	buf := make([]byte, bufferSize)
+	buf := make([]byte, c.GetNetworkBufferSize())
 	io.CopyBuffer(w, resp.Body, buf)
 }
+
+func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
+	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
+		return realdebrid.UnrestrictLink(c.GetToken(), link)
+	}
+	resp := realdebrid.RetryUntilOk(unrestrictFn)
+	if resp == nil {
+		http.Error(w, "REAL-DEBRID IS DOWN", http.StatusInternalServerError)
+		return
+	}
+	streamFileToResponse(resp.Download, w, r, c, log)
+}
diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go
index e697eba..ca08b21 100644
--- a/pkg/realdebrid/types.go
+++ b/pkg/realdebrid/types.go
@@ -11,11 +11,12 @@ type FileJSON struct {
 }
 
 type UnrestrictResponse struct {
-	Filename string `json:"filename"`
-	Filesize int64  `json:"filesize"`
-	Link     string `json:"link"`
-	Host     string `json:"host"`
-	Download string `json:"download,omitempty"`
+	Filename   string `json:"filename"`
+	Filesize   int64  `json:"filesize"`
+	Link       string `json:"link"`
+	Host       string `json:"host"`
+	Download   string `json:"download,omitempty"`
+	Streamable int    `json:"streamable"`
 }
 
 type Torrent struct {

From a7d7c4ae0d0edaec46c6e8ea127e588824d4be5d Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 10:25:19 +0100
Subject: [PATCH 04/19] Add healthcheck

---
 Dockerfile | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c18c4a0..0e4228b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,15 +9,20 @@ COPY . .
 RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags="-s -w" -o zurg cmd/zurg/main.go
 
 # Obfuscation stage
-# FROM alpine:3 AS obfuscator
-# WORKDIR /app
-# COPY --from=builder /app/zurg .
-# RUN apk add --no-cache upx
-# RUN upx --brute zurg
+FROM alpine:3 AS obfuscator
+WORKDIR /app
+COPY --from=builder /app/zurg .
+RUN apk add --no-cache upx
+RUN upx --brute zurg
 
 # Final stage
 FROM alpine:3
 WORKDIR /app
-COPY --from=builder /app/zurg .
+COPY --from=obfuscator /app/zurg .
 RUN apk add --no-cache fuse3
+
+# Healthcheck to verify that port 9999 is available to accept connections
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
+  CMD nc -z localhost 9999 || exit 1
+
 ENTRYPOINT ["./zurg"]

From 1349de7d3fe76d7bbf20535d6c8bf08da15cef83 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 13:53:42 +0100
Subject: [PATCH 05/19] Make it work for ipv4 and ipv6

---
 cmd/zurg/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index 9399474..ed6452b 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -32,7 +32,7 @@ func main() {
 	mux := http.NewServeMux()
 	net.Router(mux, config, torrentMgr, cache)
 
-	addr := fmt.Sprintf(":%s", config.GetPort())
+	addr := fmt.Sprintf("[::]:%s", config.GetPort())
 	server := &http.Server{Addr: addr, Handler: mux}
 
 	shutdown := make(chan os.Signal, 1)

From 3cc5044bc575c0377fdcbc646e5191decab901e0 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 14:14:33 +0100
Subject: [PATCH 06/19] Allow configuring hosts

---
 README.md                | 69 ++++------------------------------------
 cmd/zurg/main.go         |  2 +-
 config.yml.example       |  3 +-
 internal/config/load.go  |  1 +
 internal/config/types.go |  8 +++++
 5 files changed, 19 insertions(+), 64 deletions(-)

diff --git a/README.md b/README.md
index 9316259..6ee8a3d 100644
--- a/README.md
+++ b/README.md
@@ -1,77 +1,22 @@
 # zurg-testing
 
-A self-hosted Real-Debrid webdav server written from scratch, alternative to rclone_rd
+A self-hosted Real-Debrid webdav server written from scratch, alternative to rclone_rd. Together with [rclone](https://rclone.org/) it can mount your Real-Debrid torrent library into your filesystem.
 
-## How to run zurg in 5 steps
+## How to run zurg in 5 steps for Plex
 
 1. Clone this repo `git clone https://github.com/debridmediamanager/zurg-testing.git`
 2. Add your token in `config.yml`
 3. `sudo mkdir -p /mnt/zurg`
 4. Run `docker compose up -d`
-5. `time ls -1R /mnt/zurg` You're done!
+5. `time ls -1R /mnt/zurg` You're done! If you do edits on your config.yml just do `docker compose restart zurg`.
 
-The server is also exposed to your localhost via port 9999. You can point [Infuse](https://firecore.com/infuse) or any webdav clients to it.
+A webdav server is also exposed to your localhost via port `9999`.
 
-> Note: I have only tested this in Mac and Linux
-
-## Why zurg? Why not rclone_rd? Why not Real-Debrid's own webdav?
+## Why zurg? Why not X?
 
 - Better performance than anything out there; changes in your library appear instantly (assuming Plex picks it up fast enough)
 - You should be able to access every file even if the torrent names are the same so if you have a lot of these, you might notice that zurg will have more files compared to others (e.g. 2 torrents named "Simpsons" but have different seasons, zurg merges all contents in that directory)
-- You can configure a flexible directory structure in `config.yml`; you can select individual torrents that should appear on a directory by the ID you see in [DMM](https://debridmediamanager.com/)
+- You can configure a flexible directory structure in `config.yml`; you can select individual torrents that should appear on a directory by the ID you see in [DMM](https://debridmediamanager.com/).
 - If you've ever experienced Plex scanner being stuck on a file and thereby freezing Plex completely, it should not happen anymore because zurg does a comprehensive check if a torrent is dead or not
 
-## config.yml
-
-You need a `config.yml` created before you can use zurg
-
-```yaml
-# Zurg configuration version
-zurg: v1
-
-token: YOUR_TOKEN_HERE
-port: 9999
-concurrent_workers: 10 # the higher the number the faster zurg runs through your library but too high and you will get rate limited
-check_for_changes_every_secs: 15 # zurg polls real-debrid for changes in your library
-info_cache_time_hours: 12 # how long do we want to check if a torrent is still alive or dead? 12 to 24 hours is good enough
-
-# zurg can repair broken links, but it doesn't mean it will appear on the same location (especially if there's only 1 episode missing)
-# e.g. i was missing e06 of Better.Call.Saul.S03.2160p.NF.WEBRip.DD5.1.x264-ViSUM
-# after zurg re-added the file, it appeared on a different directory:
-# Better.Call.Saul.S03E06.2160p.NF.WEBRip.DD5.1.x264-ViSUM.mkv
-enable_repair: false # BEWARE! THERE CAN ONLY BE 1 INSTANCE OF ZURG THAT SHOULD REPAIR YOUR TORRENTS
-
-# List of directory definitions and their filtering rules
-directories:
-  # Configuration for TV shows
-  shows:
-    group: media # directories on different groups have duplicates of the same torrent
-    filters:
-      - regex: /season[\s\.]?\d/i          # Capture torrent names with the term 'season' in any case
-      - regex: /saison[\s\.]?\d/i          # For non-English namings
-      - regex: /stagione[\s\.]?\d/i        # if there's french, there should be italian too
-      - regex: /s\d\d/i           # Capture common season notations like S01, S02, etc.
-      - regex: /\btv/i            # anything that has TV in it is a TV show, right?
-      - contains: complete
-      - contains: seasons
-
-  # Configuration for movies
-  movies:
-    group: media # because movies and shows are in the same group, and shows come first before movies, all torrents that doesn't fall into shows will fall into movies
-    filters:
-      - regex: /.*/ # you cannot leave a directory without filters because it will not have any torrents in it
-
-  "ALL MY STUFFS":
-    group: all           # notice the group now is "all", which means it will have all the torrents of shows+movies combined because this directory is alone in this group
-    filters:
-      - regex: /.*/
-
-  "Kids":
-    group: kids
-    filters:
-      - not_contains: xxx       # Ensures adult content is excluded
-      - id: XFPQ5UCMUVAEG       # Specific inclusion by torrent ID
-      - id: VDRPYNRPQHEXC
-      - id: YELNX3XR5XJQM
-
-```
+## Please read our [wiki](https://github.com/debridmediamanager/zurg-testing/wiki) for more information!
diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index ed6452b..d5b1e16 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -32,7 +32,7 @@ func main() {
 	mux := http.NewServeMux()
 	net.Router(mux, config, torrentMgr, cache)
 
-	addr := fmt.Sprintf("[::]:%s", config.GetPort())
+	addr := fmt.Sprintf("%s:%s", config.GetHost(), config.GetPort())
 	server := &http.Server{Addr: addr, Handler: mux}
 
 	shutdown := make(chan os.Signal, 1)
diff --git a/config.yml.example b/config.yml.example
index 718f7a8..c73f88c 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -2,7 +2,8 @@
 zurg: v1
 
 token: YOUR_RD_API_TOKEN # https://real-debrid.com/apitoken
-port: 9999
+host: "[::]" # do not change this if you are running it inside a docker container
+port: 9999 # do not change this if you are running it inside a docker container
 concurrent_workers: 10
 check_for_changes_every_secs: 15
 info_cache_time_hours: 12
diff --git a/internal/config/load.go b/internal/config/load.go
index 5b6a404..9cf2f1b 100644
--- a/internal/config/load.go
+++ b/internal/config/load.go
@@ -15,6 +15,7 @@ type ConfigInterface interface {
 	GetRefreshEverySeconds() int
 	GetCacheTimeHours() int
 	EnableRepair() bool
+	GetHost() string
 	GetPort() string
 	GetDirectories() []string
 	MeetsConditions(directory, torrentID, torrentName string, fileNames []string) bool
diff --git a/internal/config/types.go b/internal/config/types.go
index 0ca050a..985ef60 100644
--- a/internal/config/types.go
+++ b/internal/config/types.go
@@ -3,6 +3,7 @@ package config
 type ZurgConfig struct {
 	Version             string `yaml:"zurg"`
 	Token               string `yaml:"token"`
+	Host                string `yaml:"host"`
 	Port                string `yaml:"port"`
 	NumOfWorkers        int    `yaml:"concurrent_workers"`
 	RefreshEverySeconds int    `yaml:"check_for_changes_every_secs"`
@@ -17,6 +18,13 @@ func (z *ZurgConfig) GetToken() string {
 	return z.Token
 }
 
+func (z *ZurgConfig) GetHost() string {
+	if z.Host == "" {
+		return "[::]"
+	}
+	return z.Host
+}
+
 func (z *ZurgConfig) GetPort() string {
 	if z.Port == "" {
 		return "9999"

From 0a76fdb14b01634dbdb5b439002b5447ebd33c54 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 17:20:54 +0100
Subject: [PATCH 07/19] Add retain_folder_name_extension option

---
 config.yml.example          |  1 +
 internal/config/load.go     |  1 +
 internal/config/types.go    | 27 ++++++++++++++++-----------
 internal/dav/response.go    |  3 +--
 internal/http/response.go   |  3 +--
 internal/torrent/manager.go | 23 ++++++++++++-----------
 internal/universal/get.go   | 14 ++++++--------
 internal/universal/head.go  |  4 +---
 internal/universal/util.go  |  5 ++---
 pkg/davextra/util.go        | 34 ----------------------------------
 10 files changed, 41 insertions(+), 74 deletions(-)

diff --git a/config.yml.example b/config.yml.example
index c73f88c..527fe3b 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -11,6 +11,7 @@ enable_repair: true # BEWARE! THERE CAN ONLY BE 1 INSTANCE OF ZURG THAT SHOULD R
 on_library_update: |
   echo "hook"
 network_buffer_size: 32768 # 32KB
+retain_folder_name_extension: false # if true, the folder name will contain the file extension when it is a torrent for a single file
 
 # List of directory definitions and their filtering rules
 directories:
diff --git a/internal/config/load.go b/internal/config/load.go
index 9cf2f1b..10139bd 100644
--- a/internal/config/load.go
+++ b/internal/config/load.go
@@ -22,6 +22,7 @@ type ConfigInterface interface {
 	GetOnLibraryUpdate() string
 	GetNetworkBufferSize() int
 	GetMountPoint() string
+	EnableRetainFolderNameExtension() bool
 }
 
 func LoadZurgConfig(filename string) (ConfigInterface, error) {
diff --git a/internal/config/types.go b/internal/config/types.go
index 985ef60..ddca6a5 100644
--- a/internal/config/types.go
+++ b/internal/config/types.go
@@ -1,17 +1,18 @@
 package config
 
 type ZurgConfig struct {
-	Version             string `yaml:"zurg"`
-	Token               string `yaml:"token"`
-	Host                string `yaml:"host"`
-	Port                string `yaml:"port"`
-	NumOfWorkers        int    `yaml:"concurrent_workers"`
-	RefreshEverySeconds int    `yaml:"check_for_changes_every_secs"`
-	CacheTimeHours      int    `yaml:"info_cache_time_hours"`
-	CanRepair           bool   `yaml:"enable_repair"`
-	OnLibraryUpdate     string `yaml:"on_library_update"`
-	NetworkBufferSize   int    `yaml:"network_buffer_size"`
-	MountPoint          string `yaml:"mount_point"`
+	Version                   string `yaml:"zurg"`
+	Token                     string `yaml:"token"`
+	Host                      string `yaml:"host"`
+	Port                      string `yaml:"port"`
+	NumOfWorkers              int    `yaml:"concurrent_workers"`
+	RefreshEverySeconds       int    `yaml:"check_for_changes_every_secs"`
+	CacheTimeHours            int    `yaml:"info_cache_time_hours"`
+	CanRepair                 bool   `yaml:"enable_repair"`
+	OnLibraryUpdate           string `yaml:"on_library_update"`
+	NetworkBufferSize         int    `yaml:"network_buffer_size"`
+	MountPoint                string `yaml:"mount_point"`
+	RetainFolderNameExtension bool   `yaml:"retain_folder_name_extension"`
 }
 
 func (z *ZurgConfig) GetToken() string {
@@ -68,3 +69,7 @@ func (z *ZurgConfig) GetNetworkBufferSize() int {
 func (z *ZurgConfig) GetMountPoint() string {
 	return z.MountPoint
 }
+
+func (z *ZurgConfig) EnableRetainFolderNameExtension() bool {
+	return z.RetainFolderNameExtension
+}
diff --git a/internal/dav/response.go b/internal/dav/response.go
index 5d1cfdf..6098b6f 100644
--- a/internal/dav/response.go
+++ b/internal/dav/response.go
@@ -49,14 +49,13 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*
 	for _, torrent := range torrents {
 		for _, file := range torrent.SelectedFiles {
 			if file.Link == "" {
+				// TODO: fix the file?
 				// log.Println("File has no link, skipping (repairing links take time)", file.Path)
 				continue
 			}
 
 			filename := filepath.Base(file.Path)
 			if finalName[filename] {
-				// fragment := davextra.GetLinkFragment(file.Link)
-				// filename = davextra.InsertLinkFragment(filename, fragment)
 				continue
 			}
 			finalName[filename] = true
diff --git a/internal/http/response.go b/internal/http/response.go
index 68c0d94..f488257 100644
--- a/internal/http/response.go
+++ b/internal/http/response.go
@@ -40,14 +40,13 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (s
 	for _, torrent := range torrents {
 		for _, file := range torrent.SelectedFiles {
 			if file.Link == "" {
+				// TODO: fix the file?
 				// log.Println("File has no link, skipping", file.Path)
 				continue
 			}
 
 			filename := filepath.Base(file.Path)
 			if finalName[filename] {
-				// fragment := davextra.GetLinkFragment(file.Link)
-				// filename = davextra.InsertLinkFragment(filename, fragment)
 				continue
 			}
 			finalName[filename] = true
diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index 28c4a0a..f1f43c8 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -35,7 +35,7 @@ type TorrentManager struct {
 // and store them in-memory; it is called only once at startup
 func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
 	t := &TorrentManager{
-		requiredVersion:   "4.11.2023",
+		requiredVersion:   fmt.Sprintf("4.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
 		config:            config,
 		cache:             cache,
 		workerPool:        make(chan bool, config.GetNumOfWorkers()),
@@ -256,7 +256,7 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 		// see if api data and file data still match
 		// then it means data is still usable
 		if len(torrentFromFile.Links) == len(torrent.Links) {
-			torrent.Name = getName(torrentFromFile)
+			torrent.Name = t.getName(torrentFromFile)
 			torrent.ForRepair = torrentFromFile.ForRepair
 			torrent.SelectedFiles = torrentFromFile.SelectedFiles[:]
 			return
@@ -323,7 +323,7 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 	}
 	// update file cache
 	torrent.OriginalName = info.OriginalName
-	torrent.Name = getName(torrent)
+	torrent.Name = t.getName(torrent)
 	if len(selectedFiles) > 0 {
 		// update the torrent with more data!
 		torrent.SelectedFiles = selectedFiles
@@ -332,16 +332,17 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 	}
 }
 
-func getName(torrent *Torrent) string {
-	ret := ""
-	if torrent.OriginalName != "" {
-		ret = torrent.OriginalName
+func (t *TorrentManager) getName(torrent *Torrent) string {
+	// drop the extension from the name
+	t.log.Debugf("Original name: %s", torrent.OriginalName)
+	t.log.Debugf("Name: %s", torrent.Name)
+	if t.config.EnableRetainFolderNameExtension() && strings.Contains(torrent.Name, torrent.OriginalName) {
+		return torrent.Name
 	} else {
-		ret = torrent.Name
+		ret := strings.TrimSuffix(torrent.OriginalName, ".mp4")
+		ret = strings.TrimSuffix(ret, ".mkv")
+		return ret
 	}
-	ret = strings.TrimSuffix(ret, ".mkv")
-	ret = strings.TrimSuffix(ret, ".mp4")
-	return ret
 }
 
 // mapToDirectories maps torrents to directories
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 736f09b..8a8c3e9 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -11,7 +11,6 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/dav"
 	intHttp "github.com/debridmediamanager.com/zurg/internal/http"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
-	"github.com/debridmediamanager.com/zurg/pkg/davextra"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/hashicorp/golang-lru/v2/expirable"
@@ -58,8 +57,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		return
 	}
 
-	filenameV2, linkFragment := davextra.ExtractLinkFragment(filename)
-	torrent, file := getFile(torrents, filenameV2, linkFragment)
+	torrent, file := getFile(torrents, filename)
 	if file == nil {
 		log.Errorf("Cannot find file from path %s", requestPath)
 		http.Error(w, "File not found", http.StatusNotFound)
@@ -79,20 +77,20 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
 		if !file.Unavailable {
-			log.Errorf("Cannot unrestrict file %s %s", filenameV2, link)
+			log.Errorf("Cannot unrestrict file %s %s", filename, link)
 			t.HideTheFile(torrent, file)
 		}
 		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, c, log)
 		return
-	} else if resp.Filename != filenameV2 {
+	} else if resp.Filename != filename {
 		actualExt := filepath.Ext(resp.Filename)
-		expectedExt := filepath.Ext(filenameV2)
+		expectedExt := filepath.Ext(filename)
 		if actualExt != expectedExt && resp.Streamable != 1 {
-			log.Errorf("File extension mismatch: %s and %s", filenameV2, resp.Filename)
+			log.Errorf("File extension mismatch: %s and %s", filename, resp.Filename)
 			streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, c, log)
 			return
 		} else {
-			log.Errorf("Filename mismatch: %s and %s", filenameV2, resp.Filename)
+			log.Errorf("Filename mismatch: %s and %s", filename, resp.Filename)
 		}
 	}
 	cache.Add(requestPath, resp.Download)
diff --git a/internal/universal/head.go b/internal/universal/head.go
index 58c452a..28d3803 100644
--- a/internal/universal/head.go
+++ b/internal/universal/head.go
@@ -9,7 +9,6 @@ import (
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
-	"github.com/debridmediamanager.com/zurg/pkg/davextra"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/hashicorp/golang-lru/v2/expirable"
 )
@@ -53,8 +52,7 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren
 		return
 	}
 
-	filenameV2, linkFragment := davextra.ExtractLinkFragment(filename)
-	_, file := getFile(torrents, filenameV2, linkFragment)
+	_, file := getFile(torrents, filename)
 	if file == nil {
 		log.Errorf("Cannot find file from path %s", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
diff --git a/internal/universal/util.go b/internal/universal/util.go
index c8b4501..e9d5351 100644
--- a/internal/universal/util.go
+++ b/internal/universal/util.go
@@ -2,17 +2,16 @@ package universal
 
 import (
 	"path/filepath"
-	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 )
 
 // getFile finds a link by a fragment, it might be wrong
-func getFile(torrents []torrent.Torrent, filename, fragment string) (*torrent.Torrent, *torrent.File) {
+func getFile(torrents []torrent.Torrent, filename string) (*torrent.Torrent, *torrent.File) {
 	for t := range torrents {
 		for f, file := range torrents[t].SelectedFiles {
 			fname := filepath.Base(file.Path)
-			if filename == fname && strings.Contains(file.Link, fragment) {
+			if filename == fname {
 				return &torrents[t], &torrents[t].SelectedFiles[f]
 			}
 		}
diff --git a/pkg/davextra/util.go b/pkg/davextra/util.go
index fdda349..4b18f17 100644
--- a/pkg/davextra/util.go
+++ b/pkg/davextra/util.go
@@ -1,11 +1,7 @@
 package davextra
 
 import (
-	"fmt"
-	"net/url"
-	"path/filepath"
 	"regexp"
-	"strings"
 )
 
 // GetLinkFragment returns the longest alphanumeric string from a link
@@ -20,33 +16,3 @@ func GetLinkFragment(link string) string {
 	}
 	return longestMatch[:4]
 }
-
-// InsertLinkFragment inserts the link ID into a filename
-// returns the new filename
-func InsertLinkFragment(filename, dupeID string) string {
-	ext := filepath.Ext(filename)
-	name := strings.TrimSuffix(filename, ext)
-	return fmt.Sprintf("%s DMM%s%s", name, dupeID, ext)
-}
-
-// ExtractLinkFragment extracts the link ID from a filename
-// returns the original filename and the link ID
-func ExtractLinkFragment(filename string) (string, string) {
-	filenameV2, err := url.PathUnescape(filename)
-	if err != nil {
-		filenameV2 = filename
-	}
-	ext := filepath.Ext(filenameV2)
-	name := strings.TrimSuffix(filenameV2, ext)
-
-	r := regexp.MustCompile(`\sDMM(\w+)`)
-	matches := r.FindStringSubmatch(name)
-	if len(matches) < 2 {
-		// No ID found
-		return filenameV2, ""
-	}
-
-	// Remove ID from filename
-	originalName := strings.Replace(name, matches[0], "", 1)
-	return originalName + ext, matches[1]
-}

From 56e3540583a5473252a51c190891d0dcb9fe4bf0 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Tue, 7 Nov 2023 17:26:34 +0100
Subject: [PATCH 08/19] Remove debug logs

---
 internal/torrent/manager.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index f1f43c8..3b3b2aa 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -334,8 +334,6 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 
 func (t *TorrentManager) getName(torrent *Torrent) string {
 	// drop the extension from the name
-	t.log.Debugf("Original name: %s", torrent.OriginalName)
-	t.log.Debugf("Name: %s", torrent.Name)
 	if t.config.EnableRetainFolderNameExtension() && strings.Contains(torrent.Name, torrent.OriginalName) {
 		return torrent.Name
 	} else {

From 17cd7ae1f4a7fc0d4b3c52bad9c13bf3252b74b4 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Wed, 8 Nov 2023 07:07:55 +0100
Subject: [PATCH 09/19] Add http client for streaming

---
 go.mod                    |  6 +++-
 go.sum                    |  4 +++
 internal/universal/get.go | 64 +++++++++++++++++++++++++++------------
 3 files changed, 53 insertions(+), 21 deletions(-)

diff --git a/go.mod b/go.mod
index 69b6a67..ad5f920 100644
--- a/go.mod
+++ b/go.mod
@@ -8,4 +8,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require go.uber.org/multierr v1.10.0 // indirect
+require (
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+)
diff --git a/go.sum b/go.sum
index f49f369..5cfeeab 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,7 @@
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 8a8c3e9..62f52ae 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -6,7 +6,9 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
+	"github.com/cenkalti/backoff"
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/dav"
 	intHttp "github.com/debridmediamanager.com/zurg/internal/http"
@@ -98,41 +100,63 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 }
 
 func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
+	// Create a new request for the file download.
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
-		log.Errorf("Error creating new request %v", err)
+		log.Errorf("Error creating new request: %v", err)
 		streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, c, log)
 		return
 	}
 
+	// Copy the headers from the incoming request to the new request.
 	for k, values := range r.Header {
 		for _, v := range values {
 			req.Header.Add(k, v)
 		}
 	}
 
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		log.Errorf("Error downloading file %v", err)
+	// Create a custom HTTP client with a timeout.
+	client := &http.Client{
+		Timeout: time.Second * 30, // Full request timeout including dial, request and response.
+	}
+
+	// Define a retry policy with exponential backoff.
+	retryPolicy := backoff.NewExponentialBackOff()
+	retryPolicy.MaxElapsedTime = time.Minute * 2 // Set the maximum elapsed time for retries.
+
+	// Use an operation with retries.
+	operation := func() error {
+		resp, err := client.Do(req)
+		if err != nil {
+			log.Errorf("Error downloading file: %v", err)
+			return err
+		}
+		defer resp.Body.Close()
+
+		if resp.StatusCode >= 500 {
+			log.Errorf("Received a 5XX status code: %d", resp.StatusCode)
+			return backoff.Permanent(err) // Stop retrying on bad status code.
+		}
+
+		// Copy the headers from the response to the ResponseWriter.
+		for k, vv := range resp.Header {
+			for _, v := range vv {
+				w.Header().Add(k, v)
+			}
+		}
+
+		// Stream the content to the ResponseWriter.
+		buf := make([]byte, c.GetNetworkBufferSize())
+		_, err = io.CopyBuffer(w, resp.Body, buf)
+		return err
+	}
+
+	// Perform the operation with the retry policy.
+	if err := backoff.Retry(operation, retryPolicy); err != nil {
+		log.Errorf("Failed after retries: %v", err)
 		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, c, log)
 		return
 	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		log.Errorf("Received a nonOK status code %d", resp.StatusCode)
-		streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, c, log)
-		return
-	}
-
-	for k, vv := range resp.Header {
-		for _, v := range vv {
-			w.Header().Add(k, v)
-		}
-	}
-
-	buf := make([]byte, c.GetNetworkBufferSize())
-	io.CopyBuffer(w, resp.Body, buf)
 }
 
 func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {

From 9dfd6c32d542d1e1677478d6227c0e94dc889d29 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Wed, 8 Nov 2023 20:20:52 +0100
Subject: [PATCH 10/19] Create a torrentMap

---
 internal/config/load.go                   |   3 +-
 internal/config/v1.go                     |   3 +-
 internal/dav/listing.go                   |   3 +-
 internal/dav/response.go                  |   8 +-
 internal/http/listing.go                  |   3 +-
 internal/http/response.go                 |   8 +-
 internal/net/router.go                    |   3 +-
 internal/torrent/hooks.go                 |   3 +-
 internal/torrent/manager.go               | 281 ++++++++++++---------
 internal/torrent/types.go                 |   6 +-
 internal/universal/get.go                 |  34 +--
 internal/universal/head.go                |   3 +-
 pkg/http/client.go                        |  51 ++++
 pkg/realdebrid/api.go                     | 292 +++++++++++-----------
 pkg/realdebrid/types.go                   |  19 +-
 pkg/realdebrid/{util.go => unrestrict.go} |   9 +-
 16 files changed, 405 insertions(+), 324 deletions(-)
 create mode 100644 pkg/http/client.go
 rename pkg/realdebrid/{util.go => unrestrict.go} (85%)

diff --git a/internal/config/load.go b/internal/config/load.go
index 10139bd..1283c1a 100644
--- a/internal/config/load.go
+++ b/internal/config/load.go
@@ -26,8 +26,7 @@ type ConfigInterface interface {
 }
 
 func LoadZurgConfig(filename string) (ConfigInterface, error) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("config")
+	log := logutil.NewLogger().Named("config")
 
 	log.Debug("Loading config file ", filename)
 	content, err := os.ReadFile(filename)
diff --git a/internal/config/v1.go b/internal/config/v1.go
index a199854..85fbfb2 100644
--- a/internal/config/v1.go
+++ b/internal/config/v1.go
@@ -33,8 +33,7 @@ func (z *ZurgConfigV1) GetDirectories() []string {
 }
 
 func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("config")
+	log := logutil.NewLogger().Named("config")
 
 	var groupMap = make(map[string][]string)
 	var groupOrderMap = make(map[string]int) // To store GroupOrder for each directory
diff --git a/internal/dav/listing.go b/internal/dav/listing.go
index d2653e6..805017f 100644
--- a/internal/dav/listing.go
+++ b/internal/dav/listing.go
@@ -15,8 +15,7 @@ import (
 )
 
 func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("dav")
+	log := logutil.NewLogger().Named("dav")
 
 	requestPath := path.Clean(r.URL.Path)
 	requestPath = strings.Trim(requestPath, "/")
diff --git a/internal/dav/response.go b/internal/dav/response.go
index 6098b6f..c89d623 100644
--- a/internal/dav/response.go
+++ b/internal/dav/response.go
@@ -18,12 +18,12 @@ func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (*d
 		if item.Progress != 100 {
 			continue
 		}
-		if _, exists := seen[item.Name]; exists {
+		if _, exists := seen[item.AccessKey]; exists {
 			continue
 		}
-		seen[item.Name] = true
+		seen[item.AccessKey] = true
 
-		path := filepath.Join(basePath, item.Name)
+		path := filepath.Join(basePath, item.AccessKey)
 		responses = append(responses, dav.Directory(path))
 	}
 
@@ -39,7 +39,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*
 	var responses []dav.Response
 
 	// initial response is the directory itself
-	currentPath := filepath.Join(basePath, torrents[0].Name)
+	currentPath := filepath.Join(basePath, torrents[0].AccessKey)
 	responses = append(responses, dav.Directory(currentPath))
 
 	finalName := make(map[string]bool)
diff --git a/internal/http/listing.go b/internal/http/listing.go
index f641062..24746a3 100644
--- a/internal/http/listing.go
+++ b/internal/http/listing.go
@@ -14,8 +14,7 @@ import (
 )
 
 func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("http")
+	log := logutil.NewLogger().Named("http")
 
 	requestPath := path.Clean(r.URL.Path)
 
diff --git a/internal/http/response.go b/internal/http/response.go
index f488257..13e0ec9 100644
--- a/internal/http/response.go
+++ b/internal/http/response.go
@@ -18,13 +18,13 @@ func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (st
 		if item.Progress != 100 {
 			continue
 		}
-		if _, exists := seen[item.Name]; exists {
+		if _, exists := seen[item.AccessKey]; exists {
 			continue
 		}
-		seen[item.Name] = true
+		seen[item.AccessKey] = true
 
-		path := filepath.Join(basePath, url.PathEscape(item.Name))
-		htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", path, item.Name)
+		path := filepath.Join(basePath, url.PathEscape(item.AccessKey))
+		htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", path, item.AccessKey)
 	}
 
 	return htmlDoc, nil
diff --git a/internal/net/router.go b/internal/net/router.go
index 1c5f4fc..08cd26b 100644
--- a/internal/net/router.go
+++ b/internal/net/router.go
@@ -16,8 +16,7 @@ import (
 
 // Router creates a WebDAV router
 func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentManager, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("net")
+	log := logutil.NewLogger().Named("net")
 
 	mux.HandleFunc("/http/", func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
diff --git a/internal/torrent/hooks.go b/internal/torrent/hooks.go
index 9015f24..d149f07 100644
--- a/internal/torrent/hooks.go
+++ b/internal/torrent/hooks.go
@@ -30,8 +30,7 @@ func (se *ScriptExecutor) Execute() (string, error) {
 }
 
 func OnLibraryUpdateHook(config config.ConfigInterface) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("hooks")
+	log := logutil.NewLogger().Named("hooks")
 
 	executor := &ScriptExecutor{
 		Script: config.GetOnLibraryUpdate(),
diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index 3b3b2aa..e6b0780 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -18,7 +18,9 @@ import (
 
 type TorrentManager struct {
 	requiredVersion   string
+	rd                *realdebrid.RealDebrid
 	torrents          []Torrent
+	torrentMap        map[string]*Torrent
 	inProgress        []string
 	checksum          string
 	config            config.ConfigInterface
@@ -35,7 +37,9 @@ type TorrentManager struct {
 // and store them in-memory; it is called only once at startup
 func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
 	t := &TorrentManager{
-		requiredVersion:   fmt.Sprintf("4.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
+		requiredVersion:   fmt.Sprintf("8.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
+		rd:                realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
+		torrentMap:        make(map[string]*Torrent),
 		config:            config,
 		cache:             cache,
 		workerPool:        make(chan bool, config.GetNumOfWorkers()),
@@ -45,32 +49,49 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 		log:               logutil.NewLogger().Named("manager"),
 	}
 
-	// Initialize torrents for the first time
+	// start with a clean slate
 	t.mu.Lock()
-	t.torrents = t.getFreshListFromAPI()
-	t.checksum = t.getChecksum()
-	t.mu.Unlock()
+	t.cache.Purge()
+	t.torrents = nil
 
-	// log.Println("First checksum", t.checksum)
+	newTorrents, _, err := t.rd.GetTorrents(0)
+	if err != nil {
+		t.log.Fatalf("Cannot get torrents: %v\n", err)
+	}
 
+	torrentsChan := make(chan *Torrent, len(newTorrents))
 	var wg sync.WaitGroup
-
-	for i := range t.torrents {
+	for i := range newTorrents {
 		wg.Add(1)
 		go func(idx int) {
 			defer wg.Done()
 			t.workerPool <- true
-			t.addMoreInfo(&t.torrents[idx])
+			torrentsChan <- t.getMoreInfo(newTorrents[idx])
 			<-t.workerPool
 		}(i)
 	}
+	wg.Wait()
+	close(torrentsChan)
+	for newTorrent := range torrentsChan {
+		if newTorrent == nil {
+			continue
+		}
+		t.torrents = append(t.torrents, *newTorrent)
+		if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
+			t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
+			t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
+			t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
+			t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+		} else {
+			t.torrentMap[newTorrent.AccessKey] = newTorrent
+		}
+	}
+	t.checksum = t.getChecksum()
+	t.mu.Unlock()
 
 	if t.config.EnableRepair() {
-		go t.repairAll(&wg)
+		go t.repairAll()
 	}
-
-	wg.Wait()
-	t.mapToDirectories()
 	go t.startRefreshJob()
 
 	return t
@@ -80,7 +101,7 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
 	var torrents []Torrent
 	for i := range t.torrents {
-		for _, dir := range t.directoryMap[t.torrents[i].Name] {
+		for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
 			if dir == directory {
 				torrents = append(torrents, t.torrents[i])
 			}
@@ -100,13 +121,18 @@ func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string)
 	var matchingTorrents []Torrent
 	torrents := t.GetByDirectory(directory)
 	for i := range torrents {
-		if torrents[i].Name == torrentName || strings.HasPrefix(torrents[i].Name, torrentName) {
+		if torrents[i].AccessKey == torrentName || strings.Contains(torrents[i].AccessKey, torrentName) {
 			matchingTorrents = append(matchingTorrents, torrents[i])
 		}
 	}
 	return matchingTorrents
 }
 
+// proxy
+func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictResponse {
+	return t.rd.UnrestrictUntilOk(link)
+}
+
 // findAllDownloadedFilesFromHash finds all files that were  with a given hash
 func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File {
 	var files []File
@@ -134,7 +160,7 @@ func (t *TorrentManager) getChecksum() string {
 
 	// GetTorrents request
 	go func() {
-		torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1)
+		torrents, totalCount, err := t.rd.GetTorrents(1)
 		if err != nil {
 			errChan <- err
 			return
@@ -144,7 +170,7 @@ func (t *TorrentManager) getChecksum() string {
 
 	// GetActiveTorrentCount request
 	go func() {
-		count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
+		count, err := t.rd.GetActiveTorrentCount()
 		if err != nil {
 			errChan <- err
 			return
@@ -186,92 +212,79 @@ func (t *TorrentManager) startRefreshJob() {
 		if checksum == t.checksum {
 			continue
 		}
+
+		t.mu.Lock()
 		t.cache.Purge()
+		t.torrents = nil
 
-		newTorrents := t.getFreshListFromAPI()
+		newTorrents, _, err := t.rd.GetTorrents(0)
+		if err != nil {
+			t.log.Errorf("Cannot get torrents: %v\n", err)
+			continue
+		}
+
+		torrentsChan := make(chan *Torrent)
 		var wg sync.WaitGroup
-
 		for i := range newTorrents {
 			wg.Add(1)
 			go func(idx int) {
 				defer wg.Done()
+				t.log.Debug(newTorrents[idx].ID)
 				t.workerPool <- true
-				t.addMoreInfo(&newTorrents[idx])
+				torrentsChan <- t.getMoreInfo(newTorrents[idx])
 				<-t.workerPool
 			}(i)
 		}
 		wg.Wait()
-
-		// apply side effects
-		t.mu.Lock()
-		t.torrents = newTorrents
+		close(torrentsChan)
+		for newTorrent := range torrentsChan {
+			if newTorrent == nil {
+				continue
+			}
+			t.torrents = append(t.torrents, *newTorrent)
+			if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
+				t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
+				t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
+				t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
+				t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+			} else {
+				t.torrentMap[newTorrent.AccessKey] = newTorrent
+			}
+		}
 		t.checksum = t.getChecksum()
 		t.mu.Unlock()
 
-		// log.Println("Checksum changed", t.checksum)
 		if t.config.EnableRepair() {
-			go t.repairAll(&wg)
+			go t.repairAll()
 		}
-		go t.mapToDirectories()
 		go OnLibraryUpdateHook(t.config)
 	}
 }
 
-// getFreshListFromAPI returns all torrents
-func (t *TorrentManager) getFreshListFromAPI() []Torrent {
-	torrents, _, err := realdebrid.GetTorrents(t.config.GetToken(), 0)
-	if err != nil {
-		t.log.Errorf("Cannot get torrents: %v\n", err)
-		return nil
-	}
-
-	// convert to own internal type without SelectedFiles yet
-	// populate inProgress
-	var torrentsV2 []Torrent
-	t.inProgress = t.inProgress[:0] // reset
-	for _, torrent := range torrents {
-		torrent.Name = strings.TrimSuffix(torrent.Name, "/")
-		torrentV2 := Torrent{
-			Torrent:       torrent,
-			SelectedFiles: nil,
-			ForRepair:     false,
-			lock:          &sync.Mutex{},
-		}
-		torrentsV2 = append(torrentsV2, torrentV2)
-
-		if torrent.Progress != 100 {
-			t.inProgress = append(t.inProgress, torrent.Hash)
-		}
-	}
-
-	t.log.Infof("Fetched %d torrents", len(torrentsV2))
-	return torrentsV2
-}
-
-// addMoreInfo updates the selected files for a torrent
-func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
+// getMoreInfo updates the selected files for a torrent
+func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
+	t.log.Info("Getting more info for", rdTorrent.ID)
 	// file cache
-	torrentFromFile := t.readFromFile(torrent.ID)
+	torrentFromFile := t.readFromFile(rdTorrent.ID)
 	if torrentFromFile != nil {
 		// see if api data and file data still match
 		// then it means data is still usable
-		if len(torrentFromFile.Links) == len(torrent.Links) {
-			torrent.Name = t.getName(torrentFromFile)
-			torrent.ForRepair = torrentFromFile.ForRepair
-			torrent.SelectedFiles = torrentFromFile.SelectedFiles[:]
-			return
+		if len(torrentFromFile.Links) == len(rdTorrent.Links) {
+			return torrentFromFile
 		}
 	}
-	// no file data yet as it is still downloading
-	if torrent.Progress != 100 {
-		return
+	t.log.Debug("Getting info for", rdTorrent.ID)
+	info, err := t.rd.GetTorrentInfo(rdTorrent.ID)
+	if err != nil {
+		t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
+		return nil
 	}
 
-	// t.log.Println("Getting info for", torrent.ID)
-	info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrent.ID)
-	if err != nil {
-		t.log.Errorf("Cannot get info: %v\n", err)
-		return
+	torrent := Torrent{
+		Version:       t.requiredVersion,
+		Torrent:       *info,
+		SelectedFiles: nil,
+		ForRepair:     false,
 	}
 
 	// SelectedFiles is a subset of Files with only the selected ones
@@ -321,23 +334,25 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 			selectedFiles[i].Link = link
 		}
 	}
+
+	torrent.ForRepair = forRepair
+	torrent.SelectedFiles = selectedFiles
+	torrent.AccessKey = t.getName(info.Name, info.OriginalName)
+
 	// update file cache
-	torrent.OriginalName = info.OriginalName
-	torrent.Name = t.getName(torrent)
 	if len(selectedFiles) > 0 {
-		// update the torrent with more data!
-		torrent.SelectedFiles = selectedFiles
-		torrent.ForRepair = forRepair
-		t.writeToFile(torrent)
+		t.writeToFile(&torrent)
 	}
+	t.log.Debugf("Got info for %s %s", torrent.ID, torrent.AccessKey)
+	return &torrent
 }
 
-func (t *TorrentManager) getName(torrent *Torrent) string {
+func (t *TorrentManager) getName(name, originalName string) string {
 	// drop the extension from the name
-	if t.config.EnableRetainFolderNameExtension() && strings.Contains(torrent.Name, torrent.OriginalName) {
-		return torrent.Name
+	if t.config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
+		return name
 	} else {
-		ret := strings.TrimSuffix(torrent.OriginalName, ".mp4")
+		ret := strings.TrimSuffix(originalName, ".mp4")
 		ret = strings.TrimSuffix(ret, ".mkv")
 		return ret
 	}
@@ -357,7 +372,7 @@ func (t *TorrentManager) mapToDirectories() {
 			for i := range t.torrents {
 				// don't process torrents that are already mapped if it is not the first run
 				alreadyMappedToGroup := false
-				for _, mappedGroup := range t.processedTorrents[t.torrents[i].Name] {
+				for _, mappedGroup := range t.processedTorrents[t.torrents[i].AccessKey] {
 					if mappedGroup == group {
 						alreadyMappedToGroup = true
 					}
@@ -371,10 +386,10 @@ func (t *TorrentManager) mapToDirectories() {
 					for _, file := range t.torrents[i].SelectedFiles {
 						filenames = append(filenames, file.Path)
 					}
-					if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].Name, filenames) {
+					if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].AccessKey, filenames) {
 						found := false
 						// check if it is already mapped to this directory
-						for _, dir := range t.directoryMap[t.torrents[i].Name] {
+						for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
 							if dir == directory {
 								found = true
 								break // it is already mapped to this directory
@@ -383,14 +398,14 @@ func (t *TorrentManager) mapToDirectories() {
 						if !found {
 							counter[directory]++
 							t.mu.Lock()
-							t.directoryMap[t.torrents[i].Name] = append(t.directoryMap[t.torrents[i].Name], directory)
+							t.directoryMap[t.torrents[i].AccessKey] = append(t.directoryMap[t.torrents[i].AccessKey], directory)
 							t.mu.Unlock()
 							break // we found a directory for this torrent, so we can stop looking for more
 						}
 					}
 				}
 				t.mu.Lock()
-				t.processedTorrents[t.torrents[i].Name] = append(t.processedTorrents[t.torrents[i].Name], group)
+				t.processedTorrents[t.torrents[i].AccessKey] = append(t.processedTorrents[t.torrents[i].AccessKey], group)
 				t.mu.Unlock()
 			}
 			sum := 0
@@ -408,6 +423,33 @@ func (t *TorrentManager) mapToDirectories() {
 	}
 }
 
+func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
+	var ret []string
+	// Map torrents to directories
+	switch t.config.GetVersion() {
+	case "v1":
+		configV1 := t.config.(*config.ZurgConfigV1)
+		groupMap := configV1.GetGroupMap()
+		// for every group, iterate over every torrent
+		// and then sprinkle/distribute the torrents to the directories of the group
+		for _, directories := range groupMap {
+			for _, directory := range directories {
+				var filenames []string
+				for _, file := range torrent.SelectedFiles {
+					filenames = append(filenames, file.Path)
+				}
+				if configV1.MeetsConditions(directory, torrent.ID, torrent.AccessKey, filenames) {
+					ret = append(ret, directory)
+					break // we found a directory for this torrent for this group, so we can stop looking for more
+				}
+			}
+		}
+	default:
+		t.log.Error("Unknown config version")
+	}
+	return ret
+}
+
 // getByID returns a torrent by its ID
 func (t *TorrentManager) getByID(torrentID string) *Torrent {
 	for i := range t.torrents {
@@ -463,19 +505,18 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 	return &torrent
 }
 
-func (t *TorrentManager) repairAll(wg *sync.WaitGroup) {
-	wg.Wait()
+func (t *TorrentManager) repairAll() {
 	for _, torrent := range t.torrents {
 		if torrent.ForRepair {
-			t.log.Infof("Issues were detected on %s %s; fixing...", torrent.ID, torrent.Name)
+			t.log.Infof("There were less links than was expected on %s %s; fixing...", torrent.ID, torrent.AccessKey)
 			t.repair(torrent.ID, torrent.SelectedFiles, true)
 		}
 		if len(torrent.Links) == 0 && torrent.Progress == 100 {
 			// If the torrent has no links
 			// and already processing repair
 			// delete it!
-			t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.Name)
-			realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
+			t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.AccessKey)
+			t.rd.DeleteTorrent(torrent.ID)
 		}
 	}
 }
@@ -518,12 +559,12 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
 		}
 	}
 	if len(missingFiles) == 0 {
-		t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.Name)
+		t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.AccessKey)
 		return
 	}
 
 	// then we repair it!
-	t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.Name)
+	t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.AccessKey)
 	// check if we can still add more downloads
 	proceed := t.canCapacityHandle()
 	if !proceed {
@@ -573,7 +614,7 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
 				t.log.Info("No other missing files left to reinsert")
 			}
 		} else {
-			t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.Name)
+			t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.AccessKey)
 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", torrent.Hash)
 			return
 		}
@@ -584,7 +625,7 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
 func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool {
 	// if missingFiles is not provided, look for missing files
 	if missingFiles == "" {
-		t.log.Info("Redownloading whole torrent", torrent.Name)
+		t.log.Info("Redownloading whole torrent", torrent.AccessKey)
 		var selection string
 		for _, file := range torrent.SelectedFiles {
 			selection += fmt.Sprintf("%d,", file.ID)
@@ -598,29 +639,29 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string,
 	}
 
 	// redownload torrent
-	resp, err := realdebrid.AddMagnetHash(t.config.GetToken(), torrent.Hash)
+	resp, err := t.rd.AddMagnetHash(torrent.Hash)
 	if err != nil {
 		t.log.Errorf("Cannot redownload torrent: %v", err)
 		return false
 	}
 	newTorrentID := resp.ID
-	err = realdebrid.SelectTorrentFiles(t.config.GetToken(), newTorrentID, missingFiles)
+	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
 	if err != nil {
 		t.log.Errorf("Cannot start redownloading: %v", err)
 	}
 
 	if deleteIfFailed {
 		if err != nil {
-			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			t.rd.DeleteTorrent(newTorrentID)
 			return false
 		}
 		time.Sleep(1 * time.Second)
 		// see if the torrent is ready
-		info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), newTorrentID)
+		info, err := t.rd.GetTorrentInfo(newTorrentID)
 		if err != nil {
 			t.log.Errorf("Cannot get info on redownloaded torrent: %v", err)
 			if deleteIfFailed {
-				realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+				t.rd.DeleteTorrent(newTorrentID)
 			}
 			return false
 		}
@@ -628,17 +669,17 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string,
 
 		if info.Progress != 100 {
 			t.log.Infof("Torrent is not cached anymore so we have to wait until completion, currently %d%%", info.Progress)
-			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			t.rd.DeleteTorrent(newTorrentID)
 			return false
 		}
 
 		if len(info.Links) != len(torrent.SelectedFiles) {
 			t.log.Infof("It didn't fix the issue, only got %d files but we need %d, undoing", len(info.Links), len(torrent.SelectedFiles))
-			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			t.rd.DeleteTorrent(newTorrentID)
 			return false
 		}
 		t.log.Info("Redownload successful, deleting old torrent")
-		realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
+		t.rd.DeleteTorrent(torrent.ID)
 	}
 	return true
 }
@@ -652,43 +693,43 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 	var wg sync.WaitGroup
 
 	// Limit concurrency
-	sem := make(chan struct{}, t.config.GetNumOfWorkers())
+	sem := make(chan bool, t.config.GetNumOfWorkers())
 
 	for _, link := range info.Links {
 		wg.Add(1)
-		sem <- struct{}{}
+		sem <- true
 		go func(lnk string) {
 			defer wg.Done()
 			defer func() { <-sem }()
-
-			unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
-				return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk)
-			}
-			resp := realdebrid.RetryUntilOk(unrestrictFn)
-			if resp != nil {
-				resultsChan <- Result{Response: resp}
-			}
+			resp := t.rd.UnrestrictUntilOk(lnk)
+			resultsChan <- Result{Response: resp}
 		}(link)
 	}
 
 	go func() {
+		t.log.Debugf("Checking %d link(s) for problematic torrent id=%s", len(info.Links), info.ID)
 		wg.Wait()
 		close(sem)
 		close(resultsChan)
+		t.log.Debugf("Closing results channel for torrent id=%s, checking...", info.ID)
 	}()
 
 	isChaotic := false
 	for result := range resultsChan {
+		if result.Response == nil {
+			continue
+		}
 		found := false
 		for i := range selectedFiles {
-			if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) {
+			if strings.Contains(selectedFiles[i].Path, result.Response.Filename) {
+				t.log.Debugf("Found a file that is in the selection for torrent id=%s: %s", info.ID, result.Response.Filename)
 				selectedFiles[i].Link = result.Response.Link
 				found = true
 			}
 		}
 		if !found {
-			// "chaos" file, we don't know where it belongs
-			isChaotic = !isStreamable(result.Response.Filename)
+			isChaotic = result.Response.Streamable == 0
+			t.log.Debugf("Found a file that is not in the selection for torrent id=%s: %s %v", info.ID, result.Response.Filename, result.Response.Streamable)
 			selectedFiles = append(selectedFiles, File{
 				File: realdebrid.File{
 					Path:     result.Response.Filename,
@@ -710,7 +751,7 @@ func (t *TorrentManager) canCapacityHandle() bool {
 	const maxDelay = 60 * time.Second
 	retryCount := 0
 	for {
-		count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
+		count, err := t.rd.GetActiveTorrentCount()
 		if err != nil {
 			t.log.Errorf("Cannot get active downloads count: %v", err)
 			if retryCount >= maxRetries {
diff --git a/internal/torrent/types.go b/internal/torrent/types.go
index 37b0156..b5dfb42 100644
--- a/internal/torrent/types.go
+++ b/internal/torrent/types.go
@@ -1,17 +1,15 @@
 package torrent
 
 import (
-	"sync"
-
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 )
 
 type Torrent struct {
-	Version string
+	AccessKey string
+	Version   string
 	realdebrid.Torrent
 	SelectedFiles []File
 	ForRepair     bool
-	lock          *sync.Mutex
 }
 
 type File struct {
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 62f52ae..d7524ba 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -14,15 +14,13 @@ import (
 	intHttp "github.com/debridmediamanager.com/zurg/internal/http"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/hashicorp/golang-lru/v2/expirable"
 	"go.uber.org/zap"
 )
 
 // HandleGetRequest handles a GET request universally for both WebDAV and HTTP
 func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("uniget")
+	log := logutil.NewLogger().Named("uniget")
 
 	requestPath := path.Clean(r.URL.Path)
 	isDav := true
@@ -44,7 +42,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		return
 	}
 	if data, exists := cache.Get(requestPath); exists {
-		streamFileToResponse(data, w, r, c, log)
+		streamFileToResponse(data, w, r, t, c, log)
 		return
 	}
 
@@ -68,43 +66,40 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	if file.Link == "" {
 		// This is a dead file, serve an alternate file
 		log.Errorf("File %s is no longer available", filename)
-		streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, c, log)
+		streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, t, c, log)
 		return
 	}
 	link := file.Link
 
-	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
-		return realdebrid.UnrestrictLink(c.GetToken(), link)
-	}
-	resp := realdebrid.RetryUntilOk(unrestrictFn)
+	resp := t.UnrestrictUntilOk(link)
 	if resp == nil {
 		if !file.Unavailable {
 			log.Errorf("Cannot unrestrict file %s %s", filename, link)
 			t.HideTheFile(torrent, file)
 		}
-		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, c, log)
+		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
 		return
 	} else if resp.Filename != filename {
 		actualExt := filepath.Ext(resp.Filename)
 		expectedExt := filepath.Ext(filename)
 		if actualExt != expectedExt && resp.Streamable != 1 {
 			log.Errorf("File extension mismatch: %s and %s", filename, resp.Filename)
-			streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, c, log)
+			streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, t, c, log)
 			return
 		} else {
 			log.Errorf("Filename mismatch: %s and %s", filename, resp.Filename)
 		}
 	}
 	cache.Add(requestPath, resp.Download)
-	streamFileToResponse(resp.Download, w, r, c, log)
+	streamFileToResponse(resp.Download, w, r, t, c, log)
 }
 
-func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
+func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
 	// Create a new request for the file download.
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		log.Errorf("Error creating new request: %v", err)
-		streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, c, log)
+		streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, t, c, log)
 		return
 	}
 
@@ -154,19 +149,16 @@ func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c
 	// Perform the operation with the retry policy.
 	if err := backoff.Retry(operation, retryPolicy); err != nil {
 		log.Errorf("Failed after retries: %v", err)
-		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, c, log)
+		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, t, c, log)
 		return
 	}
 }
 
-func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
-	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
-		return realdebrid.UnrestrictLink(c.GetToken(), link)
-	}
-	resp := realdebrid.RetryUntilOk(unrestrictFn)
+func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
+	resp := t.UnrestrictUntilOk(link)
 	if resp == nil {
 		http.Error(w, "REAL-DEBRID IS DOWN", http.StatusInternalServerError)
 		return
 	}
-	streamFileToResponse(resp.Download, w, r, c, log)
+	streamFileToResponse(resp.Download, w, r, t, c, log)
 }
diff --git a/internal/universal/head.go b/internal/universal/head.go
index 28d3803..65211fe 100644
--- a/internal/universal/head.go
+++ b/internal/universal/head.go
@@ -14,8 +14,7 @@ import (
 )
 
 func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("unihead")
+	log := logutil.NewLogger().Named("unihead")
 
 	requestPath := path.Clean(r.URL.Path)
 	requestPath = strings.Replace(requestPath, "/http", "", 1)
diff --git a/pkg/http/client.go b/pkg/http/client.go
new file mode 100644
index 0000000..50b65ad
--- /dev/null
+++ b/pkg/http/client.go
@@ -0,0 +1,51 @@
+package http
+
+import (
+	"net/http"
+	"time"
+)
+
+type HTTPClient struct {
+	Client          *http.Client
+	MaxRetries      int
+	Backoff         func(attempt int) time.Duration
+	CheckRespStatus func(resp *http.Response, err error) bool
+	BearerToken     string
+}
+
+func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
+	if r.BearerToken != "" {
+		req.Header.Set("Authorization", "Bearer "+r.BearerToken)
+	}
+	var resp *http.Response
+	var err error
+	for attempt := 0; attempt < r.MaxRetries; attempt++ {
+		resp, err = r.Client.Do(req)
+		if !r.CheckRespStatus(resp, err) {
+			return resp, err
+		}
+		time.Sleep(r.Backoff(attempt))
+	}
+	return resp, err
+}
+
+func NewHTTPClient(token string, maxRetries int, timeout time.Duration) *HTTPClient {
+	return &HTTPClient{
+		BearerToken: token,
+		Client:      &http.Client{Timeout: timeout},
+		MaxRetries:  maxRetries,
+		Backoff: func(attempt int) time.Duration {
+			return time.Duration(attempt) * time.Second
+		},
+		CheckRespStatus: func(resp *http.Response, err error) bool {
+			if err != nil {
+				return true
+			}
+			if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+				return true
+			}
+			// no need to retry because the status code is 2XX
+			return false
+		},
+	}
+}
diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go
index 3d765b2..f808f64 100644
--- a/pkg/realdebrid/api.go
+++ b/pkg/realdebrid/api.go
@@ -3,94 +3,77 @@ package realdebrid
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
+	"time"
+
+	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
+	"go.uber.org/zap"
 )
 
-func UnrestrictCheck(accessToken, link string) (*UnrestrictResponse, error) {
+type RealDebrid struct {
+	log    *zap.SugaredLogger
+	client *zurghttp.HTTPClient
+}
+
+func NewRealDebrid(accessToken string, log *zap.SugaredLogger) *RealDebrid {
+	maxRetries := 10
+	timeout := 10 * time.Second
+	client := zurghttp.NewHTTPClient(accessToken, maxRetries, timeout)
+	log.Debugf("Created an HTTP client with %d max retries and %s timeout", maxRetries, timeout)
+	return &RealDebrid{
+		log:    log,
+		client: client,
+	}
+}
+
+func (rd *RealDebrid) UnrestrictCheck(link string) (*UnrestrictResponse, error) {
 	data := url.Values{}
 	data.Set("link", link)
 
 	req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/check", bytes.NewBufferString(data.Encode()))
 	if err != nil {
+		rd.log.Errorf("Error when creating a unrestrict check request: %v", err)
 		return nil, err
 	}
-
-	req.Header.Set("Authorization", "Bearer "+accessToken)
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := rd.client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the unrestrict check request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
+		rd.log.Errorf("Error when reading the body of unrestrict check response: %v", err)
 		return nil, err
 	}
 
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		rd.log.Errorf("Received a %s from unrestrict check endpoint", resp.Status)
 		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
 	}
 
 	var response UnrestrictResponse
 	err = json.Unmarshal(body, &response)
 	if err != nil {
+		rd.log.Errorf("Error when decoding unrestrict check JSON: %v", err)
 		return nil, err
 	}
 
-	return &response, nil
-}
-
-func UnrestrictLink(accessToken, link string) (*UnrestrictResponse, error) {
-	data := url.Values{}
-	data.Set("link", link)
-
-	req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/link", bytes.NewBufferString(data.Encode()))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
-	var response UnrestrictResponse
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return nil, err
-	}
-
-	if !canFetchFirstByte(response.Download) {
-		return nil, fmt.Errorf("can't fetch first byte")
-	}
+	rd.log.Info("Link %s is streamable? %v", response.Streamable)
 	return &response, nil
 }
 
 // GetTorrents returns all torrents, paginated
 // if customLimit is 0, the default limit of 2500 is used
-func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
+func (rd *RealDebrid) GetTorrents(customLimit int) ([]Torrent, int, error) {
 	baseURL := "https://api.real-debrid.com/rest/1.0/torrents"
 	var allTorrents []Torrent
 	page := 1
@@ -110,19 +93,20 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
 
 		req, err := http.NewRequest("GET", reqURL, nil)
 		if err != nil {
+			rd.log.Errorf("Error when creating a get torrents request: %v", err)
 			return nil, 0, err
 		}
 
-		req.Header.Set("Authorization", "Bearer "+accessToken)
-
-		client := &http.Client{}
-		resp, err := client.Do(req)
+		resp, err := rd.client.Do(req)
 		if err != nil {
+			rd.log.Errorf("Error when executing the get torrents request: %v", err)
 			return nil, 0, err
 		}
 		defer resp.Body.Close()
 
-		if resp.StatusCode != http.StatusOK {
+		// if status code is not 2xx, return error
+		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+			rd.log.Errorf("Received a %s from get torrents endpoint", resp.Status)
 			return nil, 0, fmt.Errorf("HTTP error: %s", resp.Status)
 		}
 
@@ -130,6 +114,7 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
 		decoder := json.NewDecoder(resp.Body)
 		err = decoder.Decode(&torrents)
 		if err != nil {
+			rd.log.Errorf("Error when decoding get torrents JSON: %v", err)
 			return nil, 0, err
 		}
 
@@ -147,126 +132,106 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
 
 		page++
 	}
-
 	return allTorrents, totalCount, nil
 }
 
-func GetTorrentInfo(accessToken, id string) (*Torrent, error) {
+func (rd *RealDebrid) GetTorrentInfo(id string) (*Torrent, error) {
 	url := "https://api.real-debrid.com/rest/1.0/torrents/info/" + id
 
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
+		rd.log.Errorf("Error when creating a get info request: %v", err)
 		return nil, err
 	}
 
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := rd.client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the get info request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
+		rd.log.Errorf("Error when reading the body of get info response: %v", err)
 		return nil, err
 	}
 
-	if resp.StatusCode != http.StatusOK {
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		rd.log.Errorf("Received a %s from get info endpoint", resp.Status)
 		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
 	}
 
 	var response Torrent
 	err = json.Unmarshal(body, &response)
 	if err != nil {
+		rd.log.Errorf("Error when : %v", err)
 		return nil, err
 	}
 
+	rd.log.Debugf("Fetched info for torrent id=%s", response.ID)
 	return &response, nil
 }
 
 // SelectTorrentFiles selects files of a torrent to start it.
-func SelectTorrentFiles(accessToken string, id string, files string) error {
-	// Prepare request data
+func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error {
 	data := url.Values{}
 	data.Set("files", files)
 
-	// Construct request URL
 	reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/selectFiles/%s", id)
 	req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode()))
 	if err != nil {
+		rd.log.Errorf("Error when creating a select files request: %v", err)
 		return err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	// Send the request
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := rd.client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the select files request: %v", err)
 		return err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
-	switch resp.StatusCode {
-	case http.StatusOK, http.StatusNoContent:
-		return nil // Success
-	case http.StatusAccepted:
-		return errors.New("action already done")
-	case http.StatusBadRequest:
-		return errors.New("bad request")
-	case http.StatusUnauthorized:
-		return errors.New("bad token (expired or invalid)")
-	case http.StatusForbidden:
-		return errors.New("permission denied (account locked or not premium)")
-	case http.StatusNotFound:
-		return errors.New("wrong parameter (invalid file id(s)) or unknown resource (invalid id)")
-	default:
-		return fmt.Errorf("unexpected HTTP error: %s", resp.Status)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		rd.log.Errorf("Received a %s from select files endpoint", resp.Status)
+		return fmt.Errorf("HTTP error: %s", resp.Status)
 	}
+
+	rd.log.Debugf("Selected files %s for torrent id=%s", len(strings.Split(files, ",")), id)
+	return nil
 }
 
 // DeleteTorrent deletes a torrent from the torrents list.
-func DeleteTorrent(accessToken string, id string) error {
+func (rd *RealDebrid) DeleteTorrent(id string) error {
 	// Construct request URL
 	reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/delete/%s", id)
 	req, err := http.NewRequest("DELETE", reqURL, nil)
 	if err != nil {
+		rd.log.Errorf("Error when creating a delete torrent request: %v", err)
 		return err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-
 	// Send the request
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := rd.client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the delete torrent request: %v", err)
 		return err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
-	switch resp.StatusCode {
-	case http.StatusNoContent:
-		return nil // Success
-	case http.StatusUnauthorized:
-		return errors.New("bad token (expired or invalid)")
-	case http.StatusForbidden:
-		return errors.New("permission denied (account locked)")
-	case http.StatusNotFound:
-		return errors.New("unknown resource")
-	default:
-		return fmt.Errorf("unexpected HTTP error: %s", resp.Status)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		rd.log.Errorf("Received a %s from delete torrent endpoint", resp.Status)
+		return fmt.Errorf("HTTP error: %s", resp.Status)
 	}
+
+	rd.log.Debugf("Deleted torrent with id=%s", id)
+	return nil
 }
 
 // AddMagnetHash adds a magnet link to download.
-func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) {
+func (rd *RealDebrid) AddMagnetHash(magnet string) (*MagnetResponse, error) {
 	// Prepare request data
 	data := url.Values{}
 	data.Set("magnet", fmt.Sprintf("magnet:?xt=urn:btih:%s", magnet))
@@ -275,77 +240,110 @@ func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) {
 	reqURL := "https://api.real-debrid.com/rest/1.0/torrents/addMagnet"
 	req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode()))
 	if err != nil {
+		rd.log.Errorf("Error when creating an add magnet request: %v", err)
 		return nil, err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	// Send the request
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := rd.client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the add magnet request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
-	switch resp.StatusCode {
-	case http.StatusCreated:
-		var response MagnetResponse
-		err := json.NewDecoder(resp.Body).Decode(&response)
-		if err != nil {
-			return nil, err
-		}
-		return &response, nil
-	case http.StatusBadRequest:
-		return nil, errors.New("bad request")
-	case http.StatusUnauthorized:
-		return nil, errors.New("bad token (expired or invalid)")
-	case http.StatusForbidden:
-		return nil, errors.New("permission denied (account locked or not premium)")
-	case http.StatusServiceUnavailable:
-		return nil, errors.New("service unavailable")
-	default:
-		return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		rd.log.Errorf("Received a %s from add magnet endpoint", resp.Status)
+		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
 	}
+
+	var response MagnetResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		rd.log.Errorf("Error when decoding add magnet JSON: %v", err)
+		return nil, err
+	}
+
+	rd.log.Debugf("Added magnet %s with id=%s", magnet, response.ID)
+	return &response, nil
 }
 
 // GetActiveTorrentCount gets the number of currently active torrents and the current maximum limit.
-func GetActiveTorrentCount(accessToken string) (*ActiveTorrentCountResponse, error) {
+func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, error) {
 	// Construct request URL
 	reqURL := "https://api.real-debrid.com/rest/1.0/torrents/activeCount"
 	req, err := http.NewRequest("GET", reqURL, nil)
 	if err != nil {
+		rd.log.Errorf("Error when creating a active torrents request: %v", err)
 		return nil, err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-
 	// Send the request
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := rd.client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the active torrents request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
-	switch resp.StatusCode {
-	case http.StatusOK:
-		var response ActiveTorrentCountResponse
-		err := json.NewDecoder(resp.Body).Decode(&response)
-		if err != nil {
-			return nil, err
-		}
-		return &response, nil
-	case http.StatusUnauthorized:
-		return nil, errors.New("bad token (expired or invalid)")
-	case http.StatusForbidden:
-		return nil, errors.New("permission denied (account locked)")
-	default:
-		return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status)
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		rd.log.Errorf("Received a %s from active torrents endpoint", resp.Status)
+		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
 	}
+
+	var response ActiveTorrentCountResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+	if err != nil {
+		rd.log.Errorf("Error when decoding active torrents JSON: %v", err)
+		return nil, err
+	}
+	return &response, nil
+}
+
+func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
+	data := url.Values{}
+	data.Set("link", link)
+
+	req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/link", bytes.NewBufferString(data.Encode()))
+	if err != nil {
+		rd.log.Errorf("Error when creating a unrestrict link request: %v", err)
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		rd.log.Errorf("Error when executing the unrestrict link request: %v", err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		rd.log.Errorf("Error when reading the body of unrestrict link response: %v", err)
+		return nil, err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		rd.log.Errorf("Received a %s from unrestrict link endpoint", resp.Status)
+		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
+	}
+
+	var response UnrestrictResponse
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		rd.log.Errorf("Error when decoding unrestrict link JSON: %v", err)
+		return nil, err
+	}
+
+	if !canFetchFirstByte(response.Download) {
+		return nil, fmt.Errorf("can't fetch first byte")
+	}
+
+	rd.log.Debugf("Unrestricted link %s into %s", link, response.Download)
+	return &response, nil
 }
diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go
index ca08b21..7f441b2 100644
--- a/pkg/realdebrid/types.go
+++ b/pkg/realdebrid/types.go
@@ -20,15 +20,16 @@ type UnrestrictResponse struct {
 }
 
 type Torrent struct {
-	ID           string   `json:"id"`
-	Name         string   `json:"filename"`
-	OriginalName string   `json:"original_filename"`
-	Hash         string   `json:"hash"`
-	Progress     int      `json:"-"`
-	Added        string   `json:"added"`
-	Bytes        int64    `json:"bytes"`
-	Links        []string `json:"links"`
-	Files        []File   `json:"files,omitempty"`
+	ID            string   `json:"id"`
+	Name          string   `json:"filename"`
+	OriginalName  string   `json:"original_filename"`
+	Hash          string   `json:"hash"`
+	Progress      int      `json:"-"`
+	Added         string   `json:"added"`
+	Bytes         int64    `json:"bytes"`
+	OriginalBytes int64    `json:"original_bytes"`
+	Links         []string `json:"links"`
+	Files         []File   `json:"files,omitempty"`
 }
 
 func (t *Torrent) UnmarshalJSON(data []byte) error {
diff --git a/pkg/realdebrid/util.go b/pkg/realdebrid/unrestrict.go
similarity index 85%
rename from pkg/realdebrid/util.go
rename to pkg/realdebrid/unrestrict.go
index e1b2f65..56f0d43 100644
--- a/pkg/realdebrid/util.go
+++ b/pkg/realdebrid/unrestrict.go
@@ -7,7 +7,14 @@ import (
 	"time"
 )
 
-func RetryUntilOk[T any](fn func() (T, error)) T {
+func (rd *RealDebrid) UnrestrictUntilOk(link string) *UnrestrictResponse {
+	unrestrictFn := func() (*UnrestrictResponse, error) {
+		return rd.UnrestrictLink(link)
+	}
+	return retryUntilOk(unrestrictFn)
+}
+
+func retryUntilOk[T any](fn func() (T, error)) T {
 	const initialDelay = 1 * time.Second
 	const maxDelay = 128 * time.Second
 	for i := 0; ; i++ {

From 15a0ba95d8530600989d59db958969f31598ca7d Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Thu, 9 Nov 2023 02:34:04 +0100
Subject: [PATCH 11/19] Refactor torrent manager

---
 cmd/zurg/main.go            |  12 +-
 go.mod                      |  11 +
 go.sum                      |  27 ++
 internal/config/v1.go       |   5 -
 internal/dav/listing.go     |  12 +-
 internal/dav/response.go    |   4 +-
 internal/http/listing.go    |  53 ++-
 internal/http/response.go   |  60 ---
 internal/net/router.go      |   4 +-
 internal/torrent/manager.go | 721 ++++++++++++++++--------------------
 internal/torrent/types.go   |  10 +-
 internal/universal/get.go   |   8 +-
 pkg/realdebrid/api.go       |   7 +-
 pkg/realdebrid/types.go     |  18 +-
 14 files changed, 436 insertions(+), 516 deletions(-)
 delete mode 100644 internal/http/response.go

diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index d5b1e16..33f9e7d 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -14,6 +14,7 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/hashicorp/golang-lru/v2/expirable"
+	"github.com/nutsdb/nutsdb"
 )
 
 func main() {
@@ -25,9 +26,18 @@ func main() {
 		log.Panicf("Config failed to load: %v", configErr)
 	}
 
+	db, err := nutsdb.Open(
+		nutsdb.DefaultOptions,
+		nutsdb.WithDir("/tmp/nutsdb"),
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+
 	cache := expirable.NewLRU[string, string](1e4, nil, time.Hour)
 
-	torrentMgr := torrent.NewTorrentManager(config, cache)
+	torrentMgr := torrent.NewTorrentManager(config, cache, db)
 
 	mux := http.NewServeMux()
 	net.Router(mux, config, torrentMgr, cache)
diff --git a/go.mod b/go.mod
index ad5f920..1b30310 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,18 @@ require (
 )
 
 require (
+	github.com/antlabs/stl v0.0.1 // indirect
+	github.com/antlabs/timer v0.0.11 // indirect
+	github.com/bwmarrin/snowflake v0.3.0 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/nutsdb/nutsdb v0.14.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/tidwall/btree v1.6.0 // indirect
+	github.com/xujiajun/mmap-go v1.0.1 // indirect
+	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 5cfeeab..6061d0f 100644
--- a/go.sum
+++ b/go.sum
@@ -1,22 +1,49 @@
+github.com/antlabs/stl v0.0.1 h1:TRD3csCrjREeLhLoQ/supaoCvFhNLBTNIwuRGrDIs6Q=
+github.com/antlabs/stl v0.0.1/go.mod h1:wvVwP1loadLG3cRjxUxK8RL4Co5xujGaZlhbztmUEqQ=
+github.com/antlabs/timer v0.0.11 h1:z75oGFLeTqJHMOcWzUPBKsBbQAz4Ske3AfqJ7bsdcwU=
+github.com/antlabs/timer v0.0.11/go.mod h1:JNV8J3yGvMKhCavGXgj9HXrVZkfdQyKCcqXBT8RdyuU=
+github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
+github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/nutsdb/nutsdb v0.14.1 h1:z+Kth/kz2oYqKmOMBZho1YK2183xjrcl6KExRtCFl18=
+github.com/nutsdb/nutsdb v0.14.1/go.mod h1:6inOji9rFBporXeHDjJny4g50RpQbkjSK5jI1hht0j8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
+github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/xujiajun/mmap-go v1.0.1 h1:7Se7ss1fLPPRW+ePgqGpCkfGIZzJV6JPq9Wq9iv/WHc=
+github.com/xujiajun/mmap-go v1.0.1/go.mod h1:CNN6Sw4SL69Sui00p0zEzcZKbt+5HtEnYUsc6BKKRMg=
+github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 h1:w0si+uee0iAaCJO9q86T6yrhdadgcsoNuh47LrUykzg=
+github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235/go.mod h1:MR4+0R6A9NS5IABnIM3384FfOq8QFVnm7WDrBOhIaMU=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/config/v1.go b/internal/config/v1.go
index 85fbfb2..dd2bf9b 100644
--- a/internal/config/v1.go
+++ b/internal/config/v1.go
@@ -5,7 +5,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"gopkg.in/yaml.v3"
 )
 
@@ -33,8 +32,6 @@ func (z *ZurgConfigV1) GetDirectories() []string {
 }
 
 func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
-	log := logutil.NewLogger().Named("config")
-
 	var groupMap = make(map[string][]string)
 	var groupOrderMap = make(map[string]int) // To store GroupOrder for each directory
 
@@ -42,7 +39,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
 	for directory, val := range z.Directories {
 		groupMap[val.Group] = append(groupMap[val.Group], directory)
 		groupOrderMap[directory] = val.GroupOrder
-		log.Debugf("Added directory to group: %s, group: %s, order: %d", directory, val.Group, val.GroupOrder)
 	}
 
 	// Sort the slice based on GroupOrder and then directory name for deterministic order
@@ -54,7 +50,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
 			return groupOrderMap[dirs[i]] < groupOrderMap[dirs[j]]
 		})
 		groupMap[group] = dirs
-		log.Debugf("Sorted directories within a group: %s %v", group, dirs)
 	}
 
 	// Return a deep copy of the map
diff --git a/internal/dav/listing.go b/internal/dav/listing.go
index 805017f..0a9dae5 100644
--- a/internal/dav/listing.go
+++ b/internal/dav/listing.go
@@ -11,22 +11,14 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/dav"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
+func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
 	log := logutil.NewLogger().Named("dav")
 
 	requestPath := path.Clean(r.URL.Path)
 	requestPath = strings.Trim(requestPath, "/")
 
-	if data, exists := cache.Get(requestPath); exists {
-		w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
-		w.WriteHeader(http.StatusMultiStatus)
-		fmt.Fprint(w, data)
-		return
-	}
-
 	var output []byte
 	var err error
 
@@ -53,8 +45,6 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 
 	if output != nil {
 		respBody := fmt.Sprintf("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n%s\n", output)
-		cache.Add(requestPath, respBody)
-
 		w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
 		w.WriteHeader(http.StatusMultiStatus)
 		fmt.Fprint(w, respBody)
diff --git a/internal/dav/response.go b/internal/dav/response.go
index c89d623..15d00b8 100644
--- a/internal/dav/response.go
+++ b/internal/dav/response.go
@@ -15,7 +15,7 @@ func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (*d
 	seen := make(map[string]bool)
 
 	for _, item := range torrents {
-		if item.Progress != 100 {
+		if item.InProgress {
 			continue
 		}
 		if _, exists := seen[item.AccessKey]; exists {
@@ -64,7 +64,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*
 			torrentResponses = append(torrentResponses, dav.File(
 				filePath,
 				file.Bytes,
-				convertRFC3339toRFC1123(torrent.Added),
+				convertRFC3339toRFC1123(torrent.LatestAdded),
 				file.Link,
 			))
 		}
diff --git a/internal/http/listing.go b/internal/http/listing.go
index 24746a3..6e39b9b 100644
--- a/internal/http/listing.go
+++ b/internal/http/listing.go
@@ -5,26 +5,19 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
+func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
 	log := logutil.NewLogger().Named("http")
 
 	requestPath := path.Clean(r.URL.Path)
 
-	if data, exists := cache.Get(requestPath); exists {
-		w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprint(w, data)
-		return
-	}
-
 	var output *string
 	var err error
 
@@ -49,8 +42,6 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
 	}
 
 	if output != nil {
-		cache.Add(requestPath, *output)
-
 		w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
 		w.WriteHeader(http.StatusOK)
 		fmt.Fprint(w, *output)
@@ -73,12 +64,19 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 
 	for _, directory := range c.GetDirectories() {
 		if basePath == directory {
-			torrents := t.GetByDirectory(basePath)
-			resp, err := createMultiTorrentResponse(requestPath, torrents)
-			if err != nil {
-				return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err)
+			htmlDoc := "<ol>"
+			for name, torrent := range t.TorrentMap {
+				if len(torrent.SelectedFiles) == 0 {
+					continue
+				}
+				for _, dir := range torrent.Directories {
+					if dir == basePath {
+						htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(name)), name)
+						break
+					}
+				}
 			}
-			return &resp, nil
+			return &htmlDoc, nil
 		}
 	}
 
@@ -86,18 +84,17 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 }
 
 func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) (*string, error) {
-	fullDir := path.Dir(requestPath)
-	directory := path.Base(fullDir)
 	torrentName := path.Base(requestPath)
-
-	sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName)
-	if len(sameNameTorrents) == 0 {
-		return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath)
+	htmlDoc := "<ol>"
+	for _, file := range t.TorrentMap[torrentName].SelectedFiles {
+		if file.Link == "" {
+			// TODO: fix the file?
+			fmt.Printf("File %s has no link, skipping\n", file.Path)
+			continue
+		}
+		filename := filepath.Base(file.Path)
+		filePath := filepath.Join(requestPath, url.PathEscape(filename))
+		htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
 	}
-
-	resp, err := createSingleTorrentResponse(requestPath, sameNameTorrents)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err)
-	}
-	return &resp, nil
+	return &htmlDoc, nil
 }
diff --git a/internal/http/response.go b/internal/http/response.go
deleted file mode 100644
index 13e0ec9..0000000
--- a/internal/http/response.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package http
-
-import (
-	"fmt"
-	"net/url"
-	"path/filepath"
-
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// createMultiTorrentResponse creates a WebDAV response for a list of torrents
-func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
-	htmlDoc := "<ul>"
-
-	seen := make(map[string]bool)
-
-	for _, item := range torrents {
-		if item.Progress != 100 {
-			continue
-		}
-		if _, exists := seen[item.AccessKey]; exists {
-			continue
-		}
-		seen[item.AccessKey] = true
-
-		path := filepath.Join(basePath, url.PathEscape(item.AccessKey))
-		htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", path, item.AccessKey)
-	}
-
-	return htmlDoc, nil
-}
-
-func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
-	htmlDoc := "<ul>"
-
-	finalName := make(map[string]bool)
-
-	currentPath := filepath.Join(basePath)
-
-	for _, torrent := range torrents {
-		for _, file := range torrent.SelectedFiles {
-			if file.Link == "" {
-				// TODO: fix the file?
-				// log.Println("File has no link, skipping", file.Path)
-				continue
-			}
-
-			filename := filepath.Base(file.Path)
-			if finalName[filename] {
-				continue
-			}
-			finalName[filename] = true
-
-			filePath := filepath.Join(currentPath, url.PathEscape(filename))
-			htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
-		}
-	}
-
-	return htmlDoc, nil
-}
diff --git a/internal/net/router.go b/internal/net/router.go
index 08cd26b..c5d8caa 100644
--- a/internal/net/router.go
+++ b/internal/net/router.go
@@ -25,7 +25,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
 			if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 {
 				universal.HandleGetRequest(w, r, t, c, cache)
 			} else {
-				intHttp.HandleDirectoryListing(w, r, t, c, cache)
+				intHttp.HandleDirectoryListing(w, r, t, c)
 			}
 
 		case http.MethodHead:
@@ -40,7 +40,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
 		case "PROPFIND":
-			dav.HandlePropfindRequest(w, r, t, c, cache)
+			dav.HandlePropfindRequest(w, r, t, c)
 
 		case http.MethodGet:
 			universal.HandleGetRequest(w, r, t, c, cache)
diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index e6b0780..21c6667 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -3,7 +3,6 @@ package torrent
 import (
 	"encoding/gob"
 	"fmt"
-	"math"
 	"os"
 	"strings"
 	"sync"
@@ -13,18 +12,18 @@ import (
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/hashicorp/golang-lru/v2/expirable"
+	"github.com/nutsdb/nutsdb"
 	"go.uber.org/zap"
 )
 
 type TorrentManager struct {
+	TorrentMap        map[string]*Torrent
 	requiredVersion   string
 	rd                *realdebrid.RealDebrid
-	torrents          []Torrent
-	torrentMap        map[string]*Torrent
-	inProgress        []string
 	checksum          string
 	config            config.ConfigInterface
 	cache             *expirable.LRU[string, string]
+	db                *nutsdb.DB
 	workerPool        chan bool
 	directoryMap      map[string][]string
 	processedTorrents map[string][]string
@@ -34,14 +33,15 @@ type TorrentManager struct {
 
 // NewTorrentManager creates a new torrent manager
 // it will fetch all torrents and their info in the background
-// and store them in-memory; it is called only once at startup
-func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
+// and store them in-memory and cached in files
+func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string], db *nutsdb.DB) *TorrentManager {
 	t := &TorrentManager{
+		TorrentMap:        make(map[string]*Torrent),
 		requiredVersion:   fmt.Sprintf("8.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
 		rd:                realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
-		torrentMap:        make(map[string]*Torrent),
 		config:            config,
 		cache:             cache,
+		db:                db,
 		workerPool:        make(chan bool, config.GetNumOfWorkers()),
 		directoryMap:      make(map[string][]string),
 		processedTorrents: make(map[string][]string),
@@ -51,8 +51,6 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 
 	// start with a clean slate
 	t.mu.Lock()
-	t.cache.Purge()
-	t.torrents = nil
 
 	newTorrents, _, err := t.rd.GetTorrents(0)
 	if err != nil {
@@ -76,46 +74,74 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 		if newTorrent == nil {
 			continue
 		}
-		t.torrents = append(t.torrents, *newTorrent)
-		if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
-			t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
-			t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
-			t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
-			t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+		if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists {
+			t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent)
 		} else {
-			t.torrentMap[newTorrent.AccessKey] = newTorrent
+			t.TorrentMap[newTorrent.AccessKey] = newTorrent
 		}
 	}
 	t.checksum = t.getChecksum()
 	t.mu.Unlock()
 
-	if t.config.EnableRepair() {
-		go t.repairAll()
-	}
+	// if t.config.EnableRepair() {
+	// 	go t.repairAll()
+	// }
 	go t.startRefreshJob()
 
 	return t
 }
 
+func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
+	merged := t1
+
+	// Merge SelectedFiles
+	fileMap := make(map[int]File)
+	for _, f := range append(t1.SelectedFiles, t2.SelectedFiles...) {
+		if _, exists := fileMap[f.ID]; !exists {
+			fileMap[f.ID] = f
+		}
+	}
+	for _, f := range fileMap {
+		merged.SelectedFiles = append(merged.SelectedFiles, f)
+	}
+
+	// Merge Instances
+	merged.Instances = append(t1.Instances, t2.Instances...)
+
+	// LatestAdded
+	if t1.LatestAdded < t2.LatestAdded {
+		merged.LatestAdded = t2.LatestAdded
+	}
+
+	// InProgress
+	for _, instance := range merged.Instances {
+		if instance.Progress != 100 {
+			merged.InProgress = true
+			break
+		}
+	}
+
+	return merged
+}
+
 // GetByDirectory returns all torrents that have a file in the specified directory
 func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
 	var torrents []Torrent
-	for i := range t.torrents {
-		for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
+	for k, v := range t.TorrentMap {
+		found := false
+		for _, dir := range v.Directories {
 			if dir == directory {
-				torrents = append(torrents, t.torrents[i])
+				found = true
+				break
 			}
 		}
+		if found {
+			torrents = append(torrents, *t.TorrentMap[k])
+		}
 	}
 	return torrents
 }
 
-// HideTheFile marks a file as deleted
-func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) {
-	file.Unavailable = true
-	t.repair(torrent.ID, torrent.SelectedFiles, false)
-}
-
 // FindAllTorrentsWithName finds all torrents in a given directory with a given name
 func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string) []Torrent {
 	var matchingTorrents []Torrent
@@ -133,26 +159,12 @@ func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictRe
 	return t.rd.UnrestrictUntilOk(link)
 }
 
-// findAllDownloadedFilesFromHash finds all files that were  with a given hash
-func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File {
-	var files []File
-	for _, torrent := range t.torrents {
-		if torrent.Hash == hash {
-			for _, file := range torrent.SelectedFiles {
-				if file.Link != "" {
-					files = append(files, file)
-				}
-			}
-		}
-	}
-	return files
-}
-
 type torrentsResponse struct {
 	torrents   []realdebrid.Torrent
 	totalCount int
 }
 
+// generates a checksum based on the number of torrents, the first torrent id and the number of active torrents
 func (t *TorrentManager) getChecksum() string {
 	torrentsChan := make(chan torrentsResponse)
 	countChan := make(chan int)
@@ -214,8 +226,6 @@ func (t *TorrentManager) startRefreshJob() {
 		}
 
 		t.mu.Lock()
-		t.cache.Purge()
-		t.torrents = nil
 
 		newTorrents, _, err := t.rd.GetTorrents(0)
 		if err != nil {
@@ -229,7 +239,6 @@ func (t *TorrentManager) startRefreshJob() {
 			wg.Add(1)
 			go func(idx int) {
 				defer wg.Done()
-				t.log.Debug(newTorrents[idx].ID)
 				t.workerPool <- true
 				torrentsChan <- t.getMoreInfo(newTorrents[idx])
 				<-t.workerPool
@@ -241,73 +250,58 @@ func (t *TorrentManager) startRefreshJob() {
 			if newTorrent == nil {
 				continue
 			}
-			t.torrents = append(t.torrents, *newTorrent)
-			if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
-				t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
-				t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
-				t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
-				t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+			if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists {
+				t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent)
 			} else {
-				t.torrentMap[newTorrent.AccessKey] = newTorrent
+				t.TorrentMap[newTorrent.AccessKey] = newTorrent
 			}
 		}
 		t.checksum = t.getChecksum()
 		t.mu.Unlock()
 
-		if t.config.EnableRepair() {
-			go t.repairAll()
-		}
+		// if t.config.EnableRepair() {
+		// 	go t.repairAll()
+		// }
 		go OnLibraryUpdateHook(t.config)
 	}
 }
 
-// getMoreInfo updates the selected files for a torrent
+// getMoreInfo gets original name, size and files for a torrent
 func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
-	t.log.Info("Getting more info for", rdTorrent.ID)
+	var info *realdebrid.TorrentInfo
+	var err error
 	// file cache
 	torrentFromFile := t.readFromFile(rdTorrent.ID)
-	if torrentFromFile != nil {
+	if torrentFromFile != nil && len(torrentFromFile.ID) > 0 && len(torrentFromFile.Links) == len(rdTorrent.Links) {
 		// see if api data and file data still match
 		// then it means data is still usable
-		if len(torrentFromFile.Links) == len(rdTorrent.Links) {
-			return torrentFromFile
+		info = torrentFromFile
+	}
+	if info == nil {
+		info, err = t.rd.GetTorrentInfo(rdTorrent.ID)
+		if err != nil {
+			t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
+			return nil
 		}
 	}
-	t.log.Debug("Getting info for", rdTorrent.ID)
-	info, err := t.rd.GetTorrentInfo(rdTorrent.ID)
-	if err != nil {
-		t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
-		return nil
-	}
-
-	torrent := Torrent{
-		Version:       t.requiredVersion,
-		Torrent:       *info,
-		SelectedFiles: nil,
-		ForRepair:     false,
-	}
-
 	// SelectedFiles is a subset of Files with only the selected ones
 	// it also has a Link field, which can be empty
 	// if it is empty, it means the file is no longer available
 	// Files+Links together are the same as SelectedFiles
 	var selectedFiles []File
-	var streamableFiles []File
+	streamableCount := 0
 	// if some Links are empty, we need to repair it
 	forRepair := false
 	for _, file := range info.Files {
 		if isStreamable(file.Path) {
-			streamableFiles = append(streamableFiles, File{
-				File: file,
-				Link: "",
-			})
+			streamableCount++
 		}
 		if file.Selected == 0 {
 			continue
 		}
 		selectedFiles = append(selectedFiles, File{
 			File: file,
-			Link: "",
+			Link: "", // no link yet
 		})
 	}
 	if len(selectedFiles) > len(info.Links) && info.Progress == 100 {
@@ -315,12 +309,12 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		// chaotic file means RD will not output the desired file selection
 		// e.g. even if we select just a single mkv, it will output a rar
 		var isChaotic bool
-		selectedFiles, isChaotic = t.organizeChaos(info, selectedFiles)
+		selectedFiles, isChaotic = t.organizeChaos(&rdTorrent, selectedFiles)
 		if isChaotic {
 			t.log.Infof("Torrent %s %s is unfixable, it's always returning an unstreamable link, ignoring", info.ID, info.Name)
 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 		} else {
-			if len(streamableFiles) > 1 {
+			if streamableCount > 1 {
 				t.log.Infof("Torrent %s %s marked for repair", info.ID, info.Name)
 				forRepair = true
 			} else {
@@ -328,22 +322,25 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 				t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 			}
 		}
-	} else {
+	} else if len(selectedFiles) > 0 {
 		// all links are still intact! good!
 		for i, link := range info.Links {
 			selectedFiles[i].Link = link
 		}
 	}
 
-	torrent.ForRepair = forRepair
-	torrent.SelectedFiles = selectedFiles
-	torrent.AccessKey = t.getName(info.Name, info.OriginalName)
-
-	// update file cache
-	if len(selectedFiles) > 0 {
-		t.writeToFile(&torrent)
+	info.ForRepair = forRepair
+	torrent := Torrent{
+		AccessKey:     t.getName(info.Name, info.OriginalName),
+		SelectedFiles: selectedFiles,
+		Directories:   t.getDirectories(info),
+		LatestAdded:   info.Added,
+		InProgress:    info.Progress != 100,
+		Instances:     []realdebrid.TorrentInfo{*info},
+	}
+	if len(selectedFiles) > 0 && torrentFromFile == nil {
+		t.writeToFile(info) // only when there are selected files, else it's useless
 	}
-	t.log.Debugf("Got info for %s %s", torrent.ID, torrent.AccessKey)
 	return &torrent
 }
 
@@ -358,72 +355,7 @@ func (t *TorrentManager) getName(name, originalName string) string {
 	}
 }
 
-// mapToDirectories maps torrents to directories
-func (t *TorrentManager) mapToDirectories() {
-	// Map torrents to directories
-	switch t.config.GetVersion() {
-	case "v1":
-		configV1 := t.config.(*config.ZurgConfigV1)
-		groupMap := configV1.GetGroupMap()
-		// for every group, iterate over every torrent
-		// and then sprinkle/distribute the torrents to the directories of the group
-		for group, directories := range groupMap {
-			counter := make(map[string]int)
-			for i := range t.torrents {
-				// don't process torrents that are already mapped if it is not the first run
-				alreadyMappedToGroup := false
-				for _, mappedGroup := range t.processedTorrents[t.torrents[i].AccessKey] {
-					if mappedGroup == group {
-						alreadyMappedToGroup = true
-					}
-				}
-				if alreadyMappedToGroup {
-					continue
-				}
-
-				for _, directory := range directories {
-					var filenames []string
-					for _, file := range t.torrents[i].SelectedFiles {
-						filenames = append(filenames, file.Path)
-					}
-					if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].AccessKey, filenames) {
-						found := false
-						// check if it is already mapped to this directory
-						for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
-							if dir == directory {
-								found = true
-								break // it is already mapped to this directory
-							}
-						}
-						if !found {
-							counter[directory]++
-							t.mu.Lock()
-							t.directoryMap[t.torrents[i].AccessKey] = append(t.directoryMap[t.torrents[i].AccessKey], directory)
-							t.mu.Unlock()
-							break // we found a directory for this torrent, so we can stop looking for more
-						}
-					}
-				}
-				t.mu.Lock()
-				t.processedTorrents[t.torrents[i].AccessKey] = append(t.processedTorrents[t.torrents[i].AccessKey], group)
-				t.mu.Unlock()
-			}
-			sum := 0
-			for _, count := range counter {
-				sum += count
-			}
-			if sum > 0 {
-				t.log.Infof("Group processing completed: %s %v total: %d", group, counter, sum)
-			} else {
-				t.log.Infof("No new additions to directory group %s", group)
-			}
-		}
-	default:
-		t.log.Error("Unknown config version")
-	}
-}
-
-func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
+func (t *TorrentManager) getDirectories(torrent *realdebrid.TorrentInfo) []string {
 	var ret []string
 	// Map torrents to directories
 	switch t.config.GetVersion() {
@@ -435,10 +367,14 @@ func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
 		for _, directories := range groupMap {
 			for _, directory := range directories {
 				var filenames []string
-				for _, file := range torrent.SelectedFiles {
+				for _, file := range torrent.Files {
+					if file.Selected == 0 {
+						continue
+					}
 					filenames = append(filenames, file.Path)
 				}
-				if configV1.MeetsConditions(directory, torrent.ID, torrent.AccessKey, filenames) {
+				accessKey := t.getName(torrent.Name, torrent.OriginalName)
+				if configV1.MeetsConditions(directory, torrent.ID, accessKey, filenames) {
 					ret = append(ret, directory)
 					break // we found a directory for this torrent for this group, so we can stop looking for more
 				}
@@ -450,18 +386,7 @@ func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
 	return ret
 }
 
-// getByID returns a torrent by its ID
-func (t *TorrentManager) getByID(torrentID string) *Torrent {
-	for i := range t.torrents {
-		if t.torrents[i].ID == torrentID {
-			return &t.torrents[i]
-		}
-	}
-	return nil
-}
-
-// writeToFile writes a torrent to a file
-func (t *TorrentManager) writeToFile(torrent *Torrent) {
+func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) {
 	filePath := fmt.Sprintf("data/%s.bin", torrent.ID)
 	file, err := os.Create(filePath)
 	if err != nil {
@@ -475,8 +400,7 @@ func (t *TorrentManager) writeToFile(torrent *Torrent) {
 	dataEncoder.Encode(torrent)
 }
 
-// readFromFile reads a torrent from a file
-func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
+func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo {
 	filePath := fmt.Sprintf("data/%s.bin", torrentID)
 	fileInfo, err := os.Stat(filePath)
 	if err != nil {
@@ -493,7 +417,7 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 	}
 	defer file.Close()
 
-	var torrent Torrent
+	var torrent realdebrid.TorrentInfo
 	dataDecoder := gob.NewDecoder(file)
 	err = dataDecoder.Decode(&torrent)
 	if err != nil {
@@ -505,185 +429,6 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 	return &torrent
 }
 
-func (t *TorrentManager) repairAll() {
-	for _, torrent := range t.torrents {
-		if torrent.ForRepair {
-			t.log.Infof("There were less links than was expected on %s %s; fixing...", torrent.ID, torrent.AccessKey)
-			t.repair(torrent.ID, torrent.SelectedFiles, true)
-		}
-		if len(torrent.Links) == 0 && torrent.Progress == 100 {
-			// If the torrent has no links
-			// and already processing repair
-			// delete it!
-			t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.AccessKey)
-			t.rd.DeleteTorrent(torrent.ID)
-		}
-	}
-}
-
-func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReinsertionFirst bool) {
-	torrent := t.getByID(torrentID)
-	if torrent == nil {
-		return
-	}
-
-	// check if it is already "being" repaired
-	found := false
-	for _, hash := range t.inProgress {
-		if hash == torrent.Hash {
-			found = true
-			break
-		}
-	}
-	if found {
-		t.log.Infof("Repair in progress, skipping %s", torrentID)
-		return
-	}
-
-	// check if it is already repaired
-	foundFiles := t.findAllDownloadedFilesFromHash(torrent.Hash)
-	var missingFiles []File
-	for _, sFile := range selectedFiles {
-		if sFile.Link == "" || sFile.Unavailable {
-			found := false
-			for _, fFile := range foundFiles {
-				// same file but different link, then yes it has been repaired
-				if sFile.Path == fFile.Path && sFile.Link != fFile.Link {
-					found = true
-					break
-				}
-			}
-			if !found {
-				missingFiles = append(missingFiles, sFile)
-			}
-		}
-	}
-	if len(missingFiles) == 0 {
-		t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.AccessKey)
-		return
-	}
-
-	// then we repair it!
-	t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.AccessKey)
-	// check if we can still add more downloads
-	proceed := t.canCapacityHandle()
-	if !proceed {
-		t.log.Error("Cannot add more torrents, exiting")
-		return
-	}
-
-	// first solution: add the same selection, maybe it can be fixed by reinsertion?
-	success := false
-	if tryReinsertionFirst {
-		success = t.reinsertTorrent(torrent, "", true)
-	}
-	if !success {
-		// if all the selected files are missing but there are other streamable files
-		var otherStreamableFileIDs []int
-		for _, file := range torrent.Files {
-			found := false
-			for _, selectedFile := range selectedFiles {
-				if selectedFile.ID == file.ID {
-					found = true
-					break
-				}
-			}
-			if !found && isStreamable(file.Path) {
-				otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID)
-			}
-		}
-		if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 {
-			// we will download 1 extra streamable file to force a redownload of the missing files
-			// or if there's only 1 missing file, we will download 1 more to prevent a rename
-			missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",")
-			t.log.Infof("Redownloading %d missing files", len(missingFiles))
-			t.reinsertTorrent(torrent, missingFilesPlus1, false)
-		} else if len(selectedFiles) > 1 {
-			// if not, last resort: add only the missing files but do it in 2 batches
-			half := len(missingFiles) / 2
-			missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
-			missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
-			if missingFiles1 != "" {
-				t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1))
-				t.reinsertTorrent(torrent, missingFiles1, false)
-			}
-			if missingFiles2 != "" {
-				t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2))
-				t.reinsertTorrent(torrent, missingFiles2, false)
-			} else {
-				t.log.Info("No other missing files left to reinsert")
-			}
-		} else {
-			t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.AccessKey)
-			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", torrent.Hash)
-			return
-		}
-		t.log.Info("Waiting for downloads to finish")
-	}
-}
-
-func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool {
-	// if missingFiles is not provided, look for missing files
-	if missingFiles == "" {
-		t.log.Info("Redownloading whole torrent", torrent.AccessKey)
-		var selection string
-		for _, file := range torrent.SelectedFiles {
-			selection += fmt.Sprintf("%d,", file.ID)
-		}
-		if selection == "" {
-			return false
-		}
-		if len(selection) > 0 {
-			missingFiles = selection[:len(selection)-1]
-		}
-	}
-
-	// redownload torrent
-	resp, err := t.rd.AddMagnetHash(torrent.Hash)
-	if err != nil {
-		t.log.Errorf("Cannot redownload torrent: %v", err)
-		return false
-	}
-	newTorrentID := resp.ID
-	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
-	if err != nil {
-		t.log.Errorf("Cannot start redownloading: %v", err)
-	}
-
-	if deleteIfFailed {
-		if err != nil {
-			t.rd.DeleteTorrent(newTorrentID)
-			return false
-		}
-		time.Sleep(1 * time.Second)
-		// see if the torrent is ready
-		info, err := t.rd.GetTorrentInfo(newTorrentID)
-		if err != nil {
-			t.log.Errorf("Cannot get info on redownloaded torrent: %v", err)
-			if deleteIfFailed {
-				t.rd.DeleteTorrent(newTorrentID)
-			}
-			return false
-		}
-		time.Sleep(1 * time.Second)
-
-		if info.Progress != 100 {
-			t.log.Infof("Torrent is not cached anymore so we have to wait until completion, currently %d%%", info.Progress)
-			t.rd.DeleteTorrent(newTorrentID)
-			return false
-		}
-
-		if len(info.Links) != len(torrent.SelectedFiles) {
-			t.log.Infof("It didn't fix the issue, only got %d files but we need %d, undoing", len(info.Links), len(torrent.SelectedFiles))
-			t.rd.DeleteTorrent(newTorrentID)
-			return false
-		}
-		t.log.Info("Redownload successful, deleting old torrent")
-		t.rd.DeleteTorrent(torrent.ID)
-	}
-	return true
-}
-
 func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) ([]File, bool) {
 	type Result struct {
 		Response *realdebrid.UnrestrictResponse
@@ -744,43 +489,235 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 	return selectedFiles, isChaotic
 }
 
-func (t *TorrentManager) canCapacityHandle() bool {
-	// max waiting time is 45 minutes
-	const maxRetries = 50
-	const baseDelay = 1 * time.Second
-	const maxDelay = 60 * time.Second
-	retryCount := 0
-	for {
-		count, err := t.rd.GetActiveTorrentCount()
-		if err != nil {
-			t.log.Errorf("Cannot get active downloads count: %v", err)
-			if retryCount >= maxRetries {
-				t.log.Error("Max retries reached. Exiting.")
-				return false
-			}
-			delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
-			if delay > maxDelay {
-				delay = maxDelay
-			}
-			time.Sleep(delay)
-			retryCount++
-			continue
-		}
+// HideTheFile marks a file as deleted
+// func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) {
+// 	file.Unavailable = true
+// 	t.repair(torrent, false)
+// }
 
-		if count.DownloadingCount < count.MaxNumberOfTorrents {
-			t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount)
-			return true
-		}
+// func (t *TorrentManager) repairAll() {
+// 	for _, torrent := range t.torrentMap {
+// 		// do not repair if:
+// 		// in progress
+// 		hasInProgress := false
+// 		for _, info := range torrent.Instances {
+// 			if info.Progress != 100 {
+// 				hasInProgress = true
+// 				break
+// 			}
+// 		}
+// 		if hasInProgress {
+// 			continue
+// 		}
+// 		// already repaired based on other instances
+// 		var missingFiles []File
+// 		for _, file := range torrent.SelectedFiles {
+// 			if file.Link == "" || file.Unavailable {
+// 				missingFiles = append(missingFiles, file)
+// 			}
+// 		}
+// 		for _, sFile := range selectedFiles {
+// 			if sFile.Link == "" || sFile.Unavailable {
+// 				found := false
+// 				for _, fFile := range foundFiles {
+// 					// same file but different link, then yes it has been repaired
+// 					if sFile.Path == fFile.Path && sFile.Link != fFile.Link {
+// 						found = true
+// 						break
+// 					}
+// 				}
+// 				if !found {
+// 					missingFiles = append(missingFiles, sFile)
+// 				}
+// 			}
+// 		}
+// 		if len(missingFiles) == 0 {
+// 			t.log.Infof("Torrent id=%s is already repaired", info.ID)
+// 			return
+// 		}
 
-		if retryCount >= maxRetries {
-			t.log.Error("Max retries reached, exiting")
-			return false
-		}
-		delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
-		if delay > maxDelay {
-			delay = maxDelay
-		}
-		time.Sleep(delay)
-		retryCount++
-	}
-}
+// 		for _, info := range torrent.Instances {
+// 			if info.Progress != 100 {
+// 				continue
+// 			}
+// 			if info.ForRepair {
+// 				t.log.Infof("There were less links than was expected on %s %s; fixing...", info.ID, info.Name)
+// 				t.repair(&info, true)
+// 				break // only repair the first one for repair and then move on
+// 			}
+// 			if len(info.Links) == 0 && info.Progress == 100 {
+// 				// If the torrent has no links
+// 				// and already processing repair
+// 				// delete it!
+// 				t.log.Infof("Deleting broken torrent id=%s as it doesn't contain any files", info.ID)
+// 				t.rd.DeleteTorrent(info.ID)
+// 			}
+// 		}
+// 	}
+// }
+
+// func (t *TorrentManager) repair(info *realdebrid.TorrentInfo, tryReinsertionFirst bool) {
+// 	// then we repair it!
+// 	t.log.Infof("Repairing torrent id=%s", info.ID)
+// 	// check if we can still add more downloads
+// 	proceed := t.canCapacityHandle()
+// 	if !proceed {
+// 		t.log.Error("Cannot add more torrents, exiting")
+// 		return
+// 	}
+
+// 	// first solution: add the same selection, maybe it can be fixed by reinsertion?
+// 	success := false
+// 	if tryReinsertionFirst {
+// 		success = t.reinsertTorrent(info, "", true)
+// 	}
+// 	if !success {
+// 		// if all the selected files are missing but there are other streamable files
+// 		var otherStreamableFileIDs []int
+// 		for _, file := range info.Files {
+// 			found := false
+// 			for _, selectedFile := range selectedFiles {
+// 				if selectedFile.ID == file.ID {
+// 					found = true
+// 					break
+// 				}
+// 			}
+// 			if !found && isStreamable(file.Path) {
+// 				otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID)
+// 			}
+// 		}
+// 		if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 {
+// 			// we will download 1 extra streamable file to force a redownload of the missing files
+// 			// or if there's only 1 missing file, we will download 1 more to prevent a rename
+// 			missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",")
+// 			t.log.Infof("Redownloading %d missing files", len(missingFiles))
+// 			t.reinsertTorrent(info, missingFilesPlus1, false)
+// 		} else if len(selectedFiles) > 1 {
+// 			// if not, last resort: add only the missing files but do it in 2 batches
+// 			half := len(missingFiles) / 2
+// 			missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
+// 			missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
+// 			if missingFiles1 != "" {
+// 				t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1))
+// 				t.reinsertTorrent(info, missingFiles1, false)
+// 			}
+// 			if missingFiles2 != "" {
+// 				t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2))
+// 				t.reinsertTorrent(info, missingFiles2, false)
+// 			} else {
+// 				t.log.Info("No other missing files left to reinsert")
+// 			}
+// 		} else {
+// 			t.log.Infof("Torrent id=%s is unfixable as the only link cached in RD is already broken", info.ID)
+// 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+// 			return
+// 		}
+// 		t.log.Info("Waiting for downloads to finish")
+// 	}
+// }
+
+// func (t *TorrentManager) reinsertTorrent(torrent *realdebrid.TorrentInfo, missingFiles string, deleteIfFailed bool) bool {
+// 	// if missingFiles is not provided, look for missing files
+// 	if missingFiles == "" {
+// 		var tmpSelection string
+// 		for _, file := range torrent.Files {
+// 			if file.Selected == 0 {
+// 				continue
+// 			}
+// 			tmpSelection += fmt.Sprintf("%d,", file.ID)
+// 		}
+// 		if tmpSelection == "" {
+// 			return false
+// 		}
+// 		if len(tmpSelection) > 0 {
+// 			missingFiles = tmpSelection[:len(tmpSelection)-1]
+// 		}
+// 	}
+
+// 	// redownload torrent
+// 	resp, err := t.rd.AddMagnetHash(torrent.Hash)
+// 	if err != nil {
+// 		t.log.Errorf("Cannot redownload torrent: %v", err)
+// 		return false
+// 	}
+// 	newTorrentID := resp.ID
+// 	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
+// 	if err != nil {
+// 		t.log.Errorf("Cannot start redownloading: %v", err)
+// 	}
+
+// 	if deleteIfFailed {
+// 		if err != nil {
+// 			t.rd.DeleteTorrent(newTorrentID)
+// 			return false
+// 		}
+// 		time.Sleep(1 * time.Second)
+// 		// see if the torrent is ready
+// 		info, err := t.rd.GetTorrentInfo(newTorrentID)
+// 		if err != nil {
+// 			t.log.Errorf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
+// 			if deleteIfFailed {
+// 				t.rd.DeleteTorrent(newTorrentID)
+// 			}
+// 			return false
+// 		}
+// 		time.Sleep(1 * time.Second)
+
+// 		if info.Progress != 100 {
+// 			t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion, currently %d%%", info.ID, info.Progress)
+// 			t.rd.DeleteTorrent(newTorrentID)
+// 			return false
+// 		}
+
+// 		missingCount := len(strings.Split(missingFiles, ","))
+// 		if len(info.Links) != missingCount {
+// 			t.log.Infof("It didn't fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount)
+// 			t.rd.DeleteTorrent(newTorrentID)
+// 			return false
+// 		}
+// 		t.log.Infof("Redownload successful id=%s, deleting old torrent id=%s", newTorrentID, torrent.ID)
+// 		t.rd.DeleteTorrent(torrent.ID)
+// 	}
+// 	return true
+// }
+
+// func (t *TorrentManager) canCapacityHandle() bool {
+// 	// max waiting time is 45 minutes
+// 	const maxRetries = 50
+// 	const baseDelay = 1 * time.Second
+// 	const maxDelay = 60 * time.Second
+// 	retryCount := 0
+// 	for {
+// 		count, err := t.rd.GetActiveTorrentCount()
+// 		if err != nil {
+// 			t.log.Errorf("Cannot get active downloads count: %v", err)
+// 			if retryCount >= maxRetries {
+// 				t.log.Error("Max retries reached. Exiting.")
+// 				return false
+// 			}
+// 			delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+// 			if delay > maxDelay {
+// 				delay = maxDelay
+// 			}
+// 			time.Sleep(delay)
+// 			retryCount++
+// 			continue
+// 		}
+
+// 		if count.DownloadingCount < count.MaxNumberOfTorrents {
+// 			t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount)
+// 			return true
+// 		}
+
+// 		if retryCount >= maxRetries {
+// 			t.log.Error("Max retries reached, exiting")
+// 			return false
+// 		}
+// 		delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+// 		if delay > maxDelay {
+// 			delay = maxDelay
+// 		}
+// 		time.Sleep(delay)
+// 		retryCount++
+// 	}
+// }
diff --git a/internal/torrent/types.go b/internal/torrent/types.go
index b5dfb42..a1e4641 100644
--- a/internal/torrent/types.go
+++ b/internal/torrent/types.go
@@ -5,11 +5,13 @@ import (
 )
 
 type Torrent struct {
-	AccessKey string
-	Version   string
-	realdebrid.Torrent
+	AccessKey     string
 	SelectedFiles []File
-	ForRepair     bool
+	Directories   []string
+	LatestAdded   string
+	InProgress    bool
+
+	Instances []realdebrid.TorrentInfo
 }
 
 type File struct {
diff --git a/internal/universal/get.go b/internal/universal/get.go
index d7524ba..4b5f95a 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -35,9 +35,9 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	// If there are less than 3 segments, return an error or adjust as needed
 	if len(segments) <= 3 {
 		if isDav {
-			dav.HandlePropfindRequest(w, r, t, c, cache)
+			dav.HandlePropfindRequest(w, r, t, c)
 		} else {
-			intHttp.HandleDirectoryListing(w, r, t, c, cache)
+			intHttp.HandleDirectoryListing(w, r, t, c)
 		}
 		return
 	}
@@ -57,7 +57,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		return
 	}
 
-	torrent, file := getFile(torrents, filename)
+	_, file := getFile(torrents, filename)
 	if file == nil {
 		log.Errorf("Cannot find file from path %s", requestPath)
 		http.Error(w, "File not found", http.StatusNotFound)
@@ -75,7 +75,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	if resp == nil {
 		if !file.Unavailable {
 			log.Errorf("Cannot unrestrict file %s %s", filename, link)
-			t.HideTheFile(torrent, file)
+			// t.HideTheFile(torrent, file)
 		}
 		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
 		return
diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go
index f808f64..6eecce7 100644
--- a/pkg/realdebrid/api.go
+++ b/pkg/realdebrid/api.go
@@ -135,7 +135,7 @@ func (rd *RealDebrid) GetTorrents(customLimit int) ([]Torrent, int, error) {
 	return allTorrents, totalCount, nil
 }
 
-func (rd *RealDebrid) GetTorrentInfo(id string) (*Torrent, error) {
+func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) {
 	url := "https://api.real-debrid.com/rest/1.0/torrents/info/" + id
 
 	req, err := http.NewRequest("GET", url, nil)
@@ -162,7 +162,7 @@ func (rd *RealDebrid) GetTorrentInfo(id string) (*Torrent, error) {
 		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
 	}
 
-	var response Torrent
+	var response TorrentInfo
 	err = json.Unmarshal(body, &response)
 	if err != nil {
 		rd.log.Errorf("Error when : %v", err)
@@ -314,8 +314,7 @@ func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
 
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := rd.client.Do(req)
 	if err != nil {
 		rd.log.Errorf("Error when executing the unrestrict link request: %v", err)
 		return nil, err
diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go
index 7f441b2..d0ce0f5 100644
--- a/pkg/realdebrid/types.go
+++ b/pkg/realdebrid/types.go
@@ -20,16 +20,28 @@ type UnrestrictResponse struct {
 }
 
 type Torrent struct {
+	ID       string   `json:"id"`
+	Name     string   `json:"filename"`
+	Hash     string   `json:"hash"`
+	Progress int      `json:"-"`
+	Added    string   `json:"added"`
+	Bytes    int64    `json:"bytes"`
+	Links    []string `json:"links"`
+}
+
+type TorrentInfo struct {
 	ID            string   `json:"id"`
 	Name          string   `json:"filename"`
-	OriginalName  string   `json:"original_filename"`
 	Hash          string   `json:"hash"`
 	Progress      int      `json:"-"`
 	Added         string   `json:"added"`
 	Bytes         int64    `json:"bytes"`
-	OriginalBytes int64    `json:"original_bytes"`
 	Links         []string `json:"links"`
-	Files         []File   `json:"files,omitempty"`
+	OriginalName  string   `json:"original_filename"` // from info
+	OriginalBytes int64    `json:"original_bytes"`    // from info
+	Files         []File   `json:"files"`             // from info
+	ForRepair     bool     `json:"-"`
+	Version       string   `json:"-"`
 }
 
 func (t *Torrent) UnmarshalJSON(data []byte) error {

From b97f859a327cacd351817964552e2dd13ecd8496 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Fri, 10 Nov 2023 19:03:07 +0100
Subject: [PATCH 12/19] Refactor with ordered maps

---
 cmd/zurg/main.go            |   2 +-
 go.mod                      |   1 +
 go.sum                      |   2 +
 internal/dav/listing.go     |  65 +++++++++---
 internal/dav/response.go    |  79 --------------
 internal/http/listing.go    |  22 ++--
 internal/torrent/manager.go | 205 ++++++++++++++++--------------------
 internal/torrent/types.go   |   6 +-
 internal/universal/get.go   |  15 ++-
 internal/universal/head.go  |  12 +--
 internal/universal/util.go  |  19 ----
 pkg/realdebrid/types.go     |   8 +-
 12 files changed, 180 insertions(+), 256 deletions(-)
 delete mode 100644 internal/dav/response.go

diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index 33f9e7d..392d7f6 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -37,7 +37,7 @@ func main() {
 
 	cache := expirable.NewLRU[string, string](1e4, nil, time.Hour)
 
-	torrentMgr := torrent.NewTorrentManager(config, cache, db)
+	torrentMgr := torrent.NewTorrentManager(config, db)
 
 	mux := http.NewServeMux()
 	net.Router(mux, config, torrentMgr, cache)
diff --git a/go.mod b/go.mod
index 1b30310..d588ff3 100644
--- a/go.mod
+++ b/go.mod
@@ -14,6 +14,7 @@ require (
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/nutsdb/nutsdb v0.14.1 // indirect
diff --git a/go.sum b/go.sum
index 6061d0f..99f954e 100644
--- a/go.sum
+++ b/go.sum
@@ -11,6 +11,8 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
+github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
diff --git a/internal/dav/listing.go b/internal/dav/listing.go
index 0a9dae5..69bbca9 100644
--- a/internal/dav/listing.go
+++ b/internal/dav/listing.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/http"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
@@ -69,10 +70,28 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 
 	for _, directory := range c.GetDirectories() {
 		if basePath == directory {
-			torrents := t.GetByDirectory(basePath)
-			resp, err := createMultiTorrentResponse("/"+basePath, torrents)
-			if err != nil {
-				return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err)
+			var responses []dav.Response
+
+			responses = append(responses, dav.Directory(basePath))
+
+			for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+				accessKey := el.Key
+				torrent := el.Value
+				if torrent.InProgress {
+					continue
+				}
+				for _, dir := range torrent.Directories {
+					if dir == basePath {
+						path := filepath.Join(basePath, accessKey)
+						responses = append(responses, dav.Directory(path))
+						break
+					}
+				}
+			}
+
+			resp := &dav.MultiStatus{
+				XMLNS:    "DAV:",
+				Response: responses,
 			}
 			return xml.Marshal(resp)
 		}
@@ -82,17 +101,37 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 }
 
 func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) {
-	directory := path.Dir(requestPath)
-	torrentName := path.Base(requestPath)
-
-	sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName)
-	if len(sameNameTorrents) == 0 {
-		return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath)
+	accessKey := path.Base(requestPath)
+	torrent, exists := t.TorrentMap.Get(accessKey)
+	if !exists {
+		return nil, fmt.Errorf("cannot find torrent %s", requestPath)
 	}
 
-	resp, err := createSingleTorrentResponse("/"+directory, sameNameTorrents)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err)
+	var responses []dav.Response
+
+	// initial response is the directory itself
+	responses = append(responses, dav.Directory(requestPath))
+
+	for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+		file := el.Value
+		if file.Link == "" {
+			// will be caught by torrent manager's repairAll
+			// just skip it for now
+			continue
+		}
+		filename := filepath.Base(file.Path)
+		filePath := filepath.Join(requestPath, filename)
+		responses = append(responses, dav.File(
+			filePath,
+			file.Bytes,
+			convertRFC3339toRFC1123(torrent.LatestAdded),
+			file.Link,
+		))
+	}
+
+	resp := &dav.MultiStatus{
+		XMLNS:    "DAV:",
+		Response: responses,
 	}
 	return xml.Marshal(resp)
 }
diff --git a/internal/dav/response.go b/internal/dav/response.go
deleted file mode 100644
index 15d00b8..0000000
--- a/internal/dav/response.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package dav
-
-import (
-	"path/filepath"
-
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-	"github.com/debridmediamanager.com/zurg/pkg/dav"
-)
-
-// createMultiTorrentResponse creates a WebDAV response for a list of torrents
-func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (*dav.MultiStatus, error) {
-	var responses []dav.Response
-	responses = append(responses, dav.Directory(basePath))
-
-	seen := make(map[string]bool)
-
-	for _, item := range torrents {
-		if item.InProgress {
-			continue
-		}
-		if _, exists := seen[item.AccessKey]; exists {
-			continue
-		}
-		seen[item.AccessKey] = true
-
-		path := filepath.Join(basePath, item.AccessKey)
-		responses = append(responses, dav.Directory(path))
-	}
-
-	return &dav.MultiStatus{
-		XMLNS:    "DAV:",
-		Response: responses,
-	}, nil
-}
-
-// createTorrentResponse creates a WebDAV response for a single torrent
-// but it also handles the case where there are many torrents with the same name
-func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*dav.MultiStatus, error) {
-	var responses []dav.Response
-
-	// initial response is the directory itself
-	currentPath := filepath.Join(basePath, torrents[0].AccessKey)
-	responses = append(responses, dav.Directory(currentPath))
-
-	finalName := make(map[string]bool)
-
-	var torrentResponses []dav.Response
-
-	for _, torrent := range torrents {
-		for _, file := range torrent.SelectedFiles {
-			if file.Link == "" {
-				// TODO: fix the file?
-				// log.Println("File has no link, skipping (repairing links take time)", file.Path)
-				continue
-			}
-
-			filename := filepath.Base(file.Path)
-			if finalName[filename] {
-				continue
-			}
-			finalName[filename] = true
-
-			filePath := filepath.Join(currentPath, filename)
-			torrentResponses = append(torrentResponses, dav.File(
-				filePath,
-				file.Bytes,
-				convertRFC3339toRFC1123(torrent.LatestAdded),
-				file.Link,
-			))
-		}
-	}
-
-	responses = append(responses, torrentResponses...)
-
-	return &dav.MultiStatus{
-		XMLNS:    "DAV:",
-		Response: responses,
-	}, nil
-}
diff --git a/internal/http/listing.go b/internal/http/listing.go
index 6e39b9b..096c8f8 100644
--- a/internal/http/listing.go
+++ b/internal/http/listing.go
@@ -65,13 +65,15 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 	for _, directory := range c.GetDirectories() {
 		if basePath == directory {
 			htmlDoc := "<ol>"
-			for name, torrent := range t.TorrentMap {
-				if len(torrent.SelectedFiles) == 0 {
+			for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+				accessKey := el.Key
+				torrent := el.Value
+				if torrent.InProgress {
 					continue
 				}
 				for _, dir := range torrent.Directories {
 					if dir == basePath {
-						htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(name)), name)
+						htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(accessKey)), accessKey)
 						break
 					}
 				}
@@ -84,12 +86,18 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 }
 
 func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) (*string, error) {
-	torrentName := path.Base(requestPath)
+	accessKey := path.Base(requestPath)
+	torrent, _ := t.TorrentMap.Get(accessKey)
+	if torrent == nil {
+		return nil, fmt.Errorf("cannot find torrent %s", requestPath)
+	}
+
 	htmlDoc := "<ol>"
-	for _, file := range t.TorrentMap[torrentName].SelectedFiles {
+	for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+		file := el.Value
 		if file.Link == "" {
-			// TODO: fix the file?
-			fmt.Printf("File %s has no link, skipping\n", file.Path)
+			// will be caught by torrent manager's repairAll
+			// just skip it for now
 			continue
 		}
 		filename := filepath.Base(file.Path)
diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index 21c6667..a1919d9 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -3,7 +3,9 @@ package torrent
 import (
 	"encoding/gob"
 	"fmt"
+	"math"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -11,18 +13,17 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
-	"github.com/hashicorp/golang-lru/v2/expirable"
+	"github.com/elliotchance/orderedmap/v2"
 	"github.com/nutsdb/nutsdb"
 	"go.uber.org/zap"
 )
 
 type TorrentManager struct {
-	TorrentMap        map[string]*Torrent
+	TorrentMap        *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
 	requiredVersion   string
 	rd                *realdebrid.RealDebrid
 	checksum          string
 	config            config.ConfigInterface
-	cache             *expirable.LRU[string, string]
 	db                *nutsdb.DB
 	workerPool        chan bool
 	directoryMap      map[string][]string
@@ -34,13 +35,12 @@ type TorrentManager struct {
 // NewTorrentManager creates a new torrent manager
 // it will fetch all torrents and their info in the background
 // and store them in-memory and cached in files
-func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string], db *nutsdb.DB) *TorrentManager {
+func NewTorrentManager(config config.ConfigInterface, db *nutsdb.DB) *TorrentManager {
 	t := &TorrentManager{
-		TorrentMap:        make(map[string]*Torrent),
-		requiredVersion:   fmt.Sprintf("8.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
+		TorrentMap:        orderedmap.NewOrderedMap[string, *Torrent](),
+		requiredVersion:   fmt.Sprintf("10.11.2023/retain=%t", config.EnableRetainFolderNameExtension()),
 		rd:                realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
 		config:            config,
-		cache:             cache,
 		db:                db,
 		workerPool:        make(chan bool, config.GetNumOfWorkers()),
 		directoryMap:      make(map[string][]string),
@@ -74,10 +74,11 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 		if newTorrent == nil {
 			continue
 		}
-		if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists {
-			t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent)
+		torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey)
+		if torrent != nil {
+			t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
 		} else {
-			t.TorrentMap[newTorrent.AccessKey] = newTorrent
+			t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
 		}
 	}
 	t.checksum = t.getChecksum()
@@ -95,15 +96,11 @@ func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
 	merged := t1
 
 	// Merge SelectedFiles
-	fileMap := make(map[int]File)
-	for _, f := range append(t1.SelectedFiles, t2.SelectedFiles...) {
-		if _, exists := fileMap[f.ID]; !exists {
-			fileMap[f.ID] = f
+	for el := t2.SelectedFiles.Front(); el != nil; el = el.Next() {
+		if _, ok := merged.SelectedFiles.Get(el.Key); !ok {
+			merged.SelectedFiles.Set(el.Key, el.Value)
 		}
 	}
-	for _, f := range fileMap {
-		merged.SelectedFiles = append(merged.SelectedFiles, f)
-	}
 
 	// Merge Instances
 	merged.Instances = append(t1.Instances, t2.Instances...)
@@ -113,7 +110,7 @@ func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
 		merged.LatestAdded = t2.LatestAdded
 	}
 
-	// InProgress
+	// InProgress - if one of the instances is in progress, then the whole torrent is in progress
 	for _, instance := range merged.Instances {
 		if instance.Progress != 100 {
 			merged.InProgress = true
@@ -124,36 +121,6 @@ func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
 	return merged
 }
 
-// GetByDirectory returns all torrents that have a file in the specified directory
-func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
-	var torrents []Torrent
-	for k, v := range t.TorrentMap {
-		found := false
-		for _, dir := range v.Directories {
-			if dir == directory {
-				found = true
-				break
-			}
-		}
-		if found {
-			torrents = append(torrents, *t.TorrentMap[k])
-		}
-	}
-	return torrents
-}
-
-// FindAllTorrentsWithName finds all torrents in a given directory with a given name
-func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string) []Torrent {
-	var matchingTorrents []Torrent
-	torrents := t.GetByDirectory(directory)
-	for i := range torrents {
-		if torrents[i].AccessKey == torrentName || strings.Contains(torrents[i].AccessKey, torrentName) {
-			matchingTorrents = append(matchingTorrents, torrents[i])
-		}
-	}
-	return matchingTorrents
-}
-
 // proxy
 func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictResponse {
 	return t.rd.UnrestrictUntilOk(link)
@@ -244,16 +211,33 @@ func (t *TorrentManager) startRefreshJob() {
 				<-t.workerPool
 			}(i)
 		}
+
+		// deletes
+		// for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+		// 	found := false
+		// 	for _, newTorrent := range newTorrents {
+		// 		if newTorrent.ID == el.Value.AccessKey {
+		// 			found = true
+		// 			break
+		// 		}
+		// 	}
+		// 	if !found {
+		// 		t.log.Infof("Torrent id=%s is no longer found", accessKey)
+		// 		t.TorrentMap.Delete(accessKey)
+		// 	}
+		// }
+
 		wg.Wait()
 		close(torrentsChan)
 		for newTorrent := range torrentsChan {
 			if newTorrent == nil {
 				continue
 			}
-			if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists {
-				t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent)
+			torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey)
+			if torrent != nil {
+				t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
 			} else {
-				t.TorrentMap[newTorrent.AccessKey] = newTorrent
+				t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
 			}
 		}
 		t.checksum = t.getChecksum()
@@ -288,7 +272,7 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 	// it also has a Link field, which can be empty
 	// if it is empty, it means the file is no longer available
 	// Files+Links together are the same as SelectedFiles
-	var selectedFiles []File
+	selectedFiles := orderedmap.NewOrderedMap[string, *File]()
 	streamableCount := 0
 	// if some Links are empty, we need to repair it
 	forRepair := false
@@ -299,22 +283,26 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		if file.Selected == 0 {
 			continue
 		}
-		selectedFiles = append(selectedFiles, File{
+		selectedFiles.Set(filepath.Base(file.Path), &File{
 			File: file,
 			Link: "", // no link yet
 		})
 	}
-	if len(selectedFiles) > len(info.Links) && info.Progress == 100 {
-		t.log.Debugf("Some links has expired for %s %s: %d selected but only %d link(s)", info.ID, info.Name, len(selectedFiles), len(info.Links))
+	if selectedFiles.Len() > len(info.Links) && info.Progress == 100 {
+		t.log.Debugf("Some links has expired for %s %s: %d selected but only %d link(s)", info.ID, info.Name, selectedFiles.Len(), len(info.Links))
 		// chaotic file means RD will not output the desired file selection
 		// e.g. even if we select just a single mkv, it will output a rar
 		var isChaotic bool
 		selectedFiles, isChaotic = t.organizeChaos(&rdTorrent, selectedFiles)
-		if isChaotic {
+		if isChaotic && selectedFiles.Len() == 1 {
 			t.log.Infof("Torrent %s %s is unfixable, it's always returning an unstreamable link, ignoring", info.ID, info.Name)
 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 		} else {
 			if streamableCount > 1 {
+				// case for repair 1: it's missing some links (or all links)
+				// if we download it as is, we might get the same file over and over again
+				// so we need to redownload it with other files selected
+				// that is why we check if there are other streamable files
 				t.log.Infof("Torrent %s %s marked for repair", info.ID, info.Name)
 				forRepair = true
 			} else {
@@ -322,10 +310,16 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 				t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 			}
 		}
-	} else if len(selectedFiles) > 0 {
+	} else if selectedFiles.Len() == len(info.Links) {
 		// all links are still intact! good!
-		for i, link := range info.Links {
-			selectedFiles[i].Link = link
+		i := 0
+		for el := selectedFiles.Front(); el != nil; el = el.Next() {
+			if i < len(info.Links) {
+				file := el.Value
+				file.Link = info.Links[i]
+				selectedFiles.Set(el.Key, file)
+				i++
+			}
 		}
 	}
 
@@ -338,7 +332,7 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		InProgress:    info.Progress != 100,
 		Instances:     []realdebrid.TorrentInfo{*info},
 	}
-	if len(selectedFiles) > 0 && torrentFromFile == nil {
+	if selectedFiles.Len() > 0 && torrentFromFile == nil {
 		t.writeToFile(info) // only when there are selected files, else it's useless
 	}
 	return &torrent
@@ -429,7 +423,7 @@ func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo
 	return &torrent
 }
 
-func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) ([]File, bool) {
+func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *orderedmap.OrderedMap[string, *File]) (*orderedmap.OrderedMap[string, *File], bool) {
 	type Result struct {
 		Response *realdebrid.UnrestrictResponse
 	}
@@ -465,84 +459,61 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 			continue
 		}
 		found := false
-		for i := range selectedFiles {
-			if strings.Contains(selectedFiles[i].Path, result.Response.Filename) {
+		for el := selectedFiles.Front(); el != nil; el = el.Next() {
+			if file, _ := selectedFiles.Get(el.Key); strings.Contains(file.Path, result.Response.Filename) {
 				t.log.Debugf("Found a file that is in the selection for torrent id=%s: %s", info.ID, result.Response.Filename)
-				selectedFiles[i].Link = result.Response.Link
+				file.Link = result.Response.Link
 				found = true
 			}
 		}
 		if !found {
-			isChaotic = result.Response.Streamable == 0
-			t.log.Debugf("Found a file that is not in the selection for torrent id=%s: %s %v", info.ID, result.Response.Filename, result.Response.Streamable)
-			selectedFiles = append(selectedFiles, File{
-				File: realdebrid.File{
-					Path:     result.Response.Filename,
-					Bytes:    result.Response.Filesize,
-					Selected: 1,
-				},
-				Link: result.Response.Link,
-			})
+			t.log.Debugf("Found a file that is NOT in the selection for torrent id=%s: %s %v", info.ID, result.Response.Filename, result.Response.Streamable)
+			if result.Response.Streamable == 1 {
+				selectedFiles.Set(filepath.Base(result.Response.Filename), &File{
+					File: realdebrid.File{
+						ID:       math.MaxInt32,
+						Path:     result.Response.Filename,
+						Bytes:    result.Response.Filesize,
+						Selected: 1,
+					},
+					Link: result.Response.Link,
+				})
+			}
 		}
 	}
 
 	return selectedFiles, isChaotic
 }
 
-// HideTheFile marks a file as deleted
-// func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) {
-// 	file.Unavailable = true
-// 	t.repair(torrent, false)
-// }
-
 // func (t *TorrentManager) repairAll() {
-// 	for _, torrent := range t.torrentMap {
-// 		// do not repair if:
-// 		// in progress
-// 		hasInProgress := false
-// 		for _, info := range torrent.Instances {
-// 			if info.Progress != 100 {
-// 				hasInProgress = true
-// 				break
-// 			}
-// 		}
-// 		if hasInProgress {
+// 	for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+// 		torrent := el.Value
+// 		// do not repair if: in progress
+// 		if torrent.InProgress {
 // 			continue
 // 		}
-// 		// already repaired based on other instances
+
 // 		var missingFiles []File
-// 		for _, file := range torrent.SelectedFiles {
-// 			if file.Link == "" || file.Unavailable {
-// 				missingFiles = append(missingFiles, file)
+// 		for el2 := torrent.SelectedFiles.Front(); el2 != nil; el2 = el2.Next() {
+// 			file, ok := torrent.SelectedFiles.Get(el2.Key)
+// 			if !ok {
+// 				continue
 // 			}
-// 		}
-// 		for _, sFile := range selectedFiles {
-// 			if sFile.Link == "" || sFile.Unavailable {
-// 				found := false
-// 				for _, fFile := range foundFiles {
-// 					// same file but different link, then yes it has been repaired
-// 					if sFile.Path == fFile.Path && sFile.Link != fFile.Link {
-// 						found = true
-// 						break
-// 					}
-// 				}
-// 				if !found {
-// 					missingFiles = append(missingFiles, sFile)
-// 				}
+// 			// check for case of repairs like
+// 			// case 1: missing links
+// 			// case 2: unrestrictable links TODO
+// 			if file.Link == "" {
+// 				missingFiles = append(missingFiles, *file)
 // 			}
 // 		}
 // 		if len(missingFiles) == 0 {
-// 			t.log.Infof("Torrent id=%s is already repaired", info.ID)
-// 			return
+// 			continue
 // 		}
 
 // 		for _, info := range torrent.Instances {
-// 			if info.Progress != 100 {
-// 				continue
-// 			}
 // 			if info.ForRepair {
 // 				t.log.Infof("There were less links than was expected on %s %s; fixing...", info.ID, info.Name)
-// 				t.repair(&info, true)
+// 				// t.repair(&info, true)
 // 				break // only repair the first one for repair and then move on
 // 			}
 // 			if len(info.Links) == 0 && info.Progress == 100 {
@@ -557,6 +528,7 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 // }
 
 // func (t *TorrentManager) repair(info *realdebrid.TorrentInfo, tryReinsertionFirst bool) {
+// 	// file.Link == "" should be repaired
 // 	// then we repair it!
 // 	t.log.Infof("Repairing torrent id=%s", info.ID)
 // 	// check if we can still add more downloads
@@ -576,6 +548,9 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 // 		var otherStreamableFileIDs []int
 // 		for _, file := range info.Files {
 // 			found := false
+// 			for el := selectedFiles.Front(); el != nil; el = el.Next() {
+
+// 			}
 // 			for _, selectedFile := range selectedFiles {
 // 				if selectedFile.ID == file.ID {
 // 					found = true
diff --git a/internal/torrent/types.go b/internal/torrent/types.go
index a1e4641..f4c35a7 100644
--- a/internal/torrent/types.go
+++ b/internal/torrent/types.go
@@ -2,11 +2,12 @@ package torrent
 
 import (
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
+	"github.com/elliotchance/orderedmap/v2"
 )
 
 type Torrent struct {
 	AccessKey     string
-	SelectedFiles []File
+	SelectedFiles *orderedmap.OrderedMap[string, *File]
 	Directories   []string
 	LatestAdded   string
 	InProgress    bool
@@ -16,6 +17,5 @@ type Torrent struct {
 
 type File struct {
 	realdebrid.File
-	Link        string
-	Unavailable bool
+	Link string
 }
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 4b5f95a..f699433 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -47,17 +47,17 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	}
 
 	baseDirectory := segments[len(segments)-3]
-	torrentName := segments[len(segments)-2]
+	accessKey := segments[len(segments)-2]
 	filename := segments[len(segments)-1]
 
-	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
-	if torrents == nil {
-		log.Errorf("Cannot find torrent %s in the directory %s", requestPath, baseDirectory)
+	torrent, _ := t.TorrentMap.Get(accessKey)
+	if torrent == nil {
+		log.Errorf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 
-	_, file := getFile(torrents, filename)
+	file, _ := torrent.SelectedFiles.Get(filename)
 	if file == nil {
 		log.Errorf("Cannot find file from path %s", requestPath)
 		http.Error(w, "File not found", http.StatusNotFound)
@@ -73,10 +73,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 
 	resp := t.UnrestrictUntilOk(link)
 	if resp == nil {
-		if !file.Unavailable {
-			log.Errorf("Cannot unrestrict file %s %s", filename, link)
-			// t.HideTheFile(torrent, file)
-		}
+		// TODO: maybe repair the torrent?
 		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
 		return
 	} else if resp.Filename != filename {
diff --git a/internal/universal/head.go b/internal/universal/head.go
index 65211fe..a623fb8 100644
--- a/internal/universal/head.go
+++ b/internal/universal/head.go
@@ -41,17 +41,17 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren
 	}
 
 	baseDirectory := segments[len(segments)-3]
-	torrentName := segments[len(segments)-2]
+	accessKey := segments[len(segments)-2]
 	filename := segments[len(segments)-1]
 
-	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
-	if torrents == nil {
-		log.Errorf("Cannot find torrent %s in the directory %s", requestPath, baseDirectory)
-		http.Error(w, "Cannot find file", http.StatusNotFound)
+	torrent, _ := t.TorrentMap.Get(accessKey)
+	if torrent == nil {
+		log.Errorf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
+		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 
-	_, file := getFile(torrents, filename)
+	file, _ := torrent.SelectedFiles.Get(filename)
 	if file == nil {
 		log.Errorf("Cannot find file from path %s", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
diff --git a/internal/universal/util.go b/internal/universal/util.go
index e9d5351..897888b 100644
--- a/internal/universal/util.go
+++ b/internal/universal/util.go
@@ -1,20 +1 @@
 package universal
-
-import (
-	"path/filepath"
-
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// getFile finds a link by a fragment, it might be wrong
-func getFile(torrents []torrent.Torrent, filename string) (*torrent.Torrent, *torrent.File) {
-	for t := range torrents {
-		for f, file := range torrents[t].SelectedFiles {
-			fname := filepath.Base(file.Path)
-			if filename == fname {
-				return &torrents[t], &torrents[t].SelectedFiles[f]
-			}
-		}
-	}
-	return nil, nil
-}
diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go
index d0ce0f5..230cf19 100644
--- a/pkg/realdebrid/types.go
+++ b/pkg/realdebrid/types.go
@@ -44,20 +44,20 @@ type TorrentInfo struct {
 	Version       string   `json:"-"`
 }
 
-func (t *Torrent) UnmarshalJSON(data []byte) error {
-	type Alias Torrent
+func (i *TorrentInfo) UnmarshalJSON(data []byte) error {
+	type Alias TorrentInfo
 	aux := &struct {
 		Progress float64 `json:"progress"`
 		*Alias
 	}{
-		Alias: (*Alias)(t),
+		Alias: (*Alias)(i),
 	}
 
 	if err := json.Unmarshal(data, &aux); err != nil {
 		return err
 	}
 
-	t.Progress = int(math.Round(aux.Progress))
+	i.Progress = int(math.Round(aux.Progress))
 	return nil
 }
 

From ab31114f2e90c8b17dc4b09fb9c7826e6b4a4e6e Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Fri, 10 Nov 2023 21:25:53 +0100
Subject: [PATCH 13/19] Couple of fixes

---
 go.mod                      |   6 +-
 go.sum                      |   6 +-
 internal/torrent/manager.go | 192 +++++++++++++++++++-----------------
 internal/universal/get.go   |  69 +++++--------
 internal/universal/util.go  |   1 -
 pkg/http/client.go          |   4 +-
 pkg/realdebrid/api.go       |   5 +-
 7 files changed, 131 insertions(+), 152 deletions(-)
 delete mode 100644 internal/universal/util.go

diff --git a/go.mod b/go.mod
index d588ff3..e5034e3 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,9 @@ module github.com/debridmediamanager.com/zurg
 go 1.21.3
 
 require (
+	github.com/elliotchance/orderedmap/v2 v2.2.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
+	github.com/nutsdb/nutsdb v0.14.1
 	go.uber.org/zap v1.26.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -12,12 +14,8 @@ require (
 	github.com/antlabs/stl v0.0.1 // indirect
 	github.com/antlabs/timer v0.0.11 // indirect
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
-	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
-	github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
-	github.com/nutsdb/nutsdb v0.14.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/xujiajun/mmap-go v1.0.1 // indirect
diff --git a/go.sum b/go.sum
index 99f954e..ff76c95 100644
--- a/go.sum
+++ b/go.sum
@@ -4,10 +4,6 @@ github.com/antlabs/timer v0.0.11 h1:z75oGFLeTqJHMOcWzUPBKsBbQAz4Ske3AfqJ7bsdcwU=
 github.com/antlabs/timer v0.0.11/go.mod h1:JNV8J3yGvMKhCavGXgj9HXrVZkfdQyKCcqXBT8RdyuU=
 github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
 github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
-github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
-github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
-github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -46,6 +42,8 @@ golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index a1919d9..a7eabc8 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -19,17 +19,15 @@ import (
 )
 
 type TorrentManager struct {
-	TorrentMap        *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
-	requiredVersion   string
-	rd                *realdebrid.RealDebrid
-	checksum          string
-	config            config.ConfigInterface
-	db                *nutsdb.DB
-	workerPool        chan bool
-	directoryMap      map[string][]string
-	processedTorrents map[string][]string
-	mu                *sync.Mutex
-	log               *zap.SugaredLogger
+	TorrentMap      *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
+	requiredVersion string
+	rd              *realdebrid.RealDebrid
+	checksum        string
+	config          config.ConfigInterface
+	db              *nutsdb.DB
+	workerPool      chan bool
+	mu              *sync.Mutex
+	log             *zap.SugaredLogger
 }
 
 // NewTorrentManager creates a new torrent manager
@@ -37,16 +35,14 @@ type TorrentManager struct {
 // and store them in-memory and cached in files
 func NewTorrentManager(config config.ConfigInterface, db *nutsdb.DB) *TorrentManager {
 	t := &TorrentManager{
-		TorrentMap:        orderedmap.NewOrderedMap[string, *Torrent](),
-		requiredVersion:   fmt.Sprintf("10.11.2023/retain=%t", config.EnableRetainFolderNameExtension()),
-		rd:                realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
-		config:            config,
-		db:                db,
-		workerPool:        make(chan bool, config.GetNumOfWorkers()),
-		directoryMap:      make(map[string][]string),
-		processedTorrents: make(map[string][]string),
-		mu:                &sync.Mutex{},
-		log:               logutil.NewLogger().Named("manager"),
+		TorrentMap:      orderedmap.NewOrderedMap[string, *Torrent](),
+		requiredVersion: "10.11.2023",
+		rd:              realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
+		config:          config,
+		db:              db,
+		workerPool:      make(chan bool, config.GetNumOfWorkers()),
+		mu:              &sync.Mutex{},
+		log:             logutil.NewLogger().Named("manager"),
 	}
 
 	// start with a clean slate
@@ -68,6 +64,7 @@ func NewTorrentManager(config config.ConfigInterface, db *nutsdb.DB) *TorrentMan
 			<-t.workerPool
 		}(i)
 	}
+	t.log.Infof("Got %d torrents", len(newTorrents))
 	wg.Wait()
 	close(torrentsChan)
 	for newTorrent := range torrentsChan {
@@ -84,9 +81,9 @@ func NewTorrentManager(config config.ConfigInterface, db *nutsdb.DB) *TorrentMan
 	t.checksum = t.getChecksum()
 	t.mu.Unlock()
 
-	// if t.config.EnableRepair() {
-	// 	go t.repairAll()
-	// }
+	if t.config.EnableRepair() {
+		go t.repairAll()
+	}
 	go t.startRefreshJob()
 
 	return t
@@ -96,6 +93,7 @@ func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
 	merged := t1
 
 	// Merge SelectedFiles
+	// iteration works!
 	for el := t2.SelectedFiles.Front(); el != nil; el = el.Next() {
 		if _, ok := merged.SelectedFiles.Get(el.Key); !ok {
 			merged.SelectedFiles.Set(el.Key, el.Value)
@@ -133,8 +131,8 @@ type torrentsResponse struct {
 
 // generates a checksum based on the number of torrents, the first torrent id and the number of active torrents
 func (t *TorrentManager) getChecksum() string {
-	torrentsChan := make(chan torrentsResponse)
-	countChan := make(chan int)
+	torrentsChan := make(chan torrentsResponse, 1)
+	countChan := make(chan int, 1)
 	errChan := make(chan error, 2) // accommodate errors from both goroutines
 
 	// GetTorrents request
@@ -199,8 +197,9 @@ func (t *TorrentManager) startRefreshJob() {
 			t.log.Errorf("Cannot get torrents: %v\n", err)
 			continue
 		}
+		t.log.Infof("Detected changes! Refreshing %d torrents", len(newTorrents))
 
-		torrentsChan := make(chan *Torrent)
+		torrentsChan := make(chan *Torrent, len(newTorrents))
 		var wg sync.WaitGroup
 		for i := range newTorrents {
 			wg.Add(1)
@@ -212,20 +211,23 @@ func (t *TorrentManager) startRefreshJob() {
 			}(i)
 		}
 
-		// deletes
-		// for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
-		// 	found := false
-		// 	for _, newTorrent := range newTorrents {
-		// 		if newTorrent.ID == el.Value.AccessKey {
-		// 			found = true
-		// 			break
-		// 		}
-		// 	}
-		// 	if !found {
-		// 		t.log.Infof("Torrent id=%s is no longer found", accessKey)
-		// 		t.TorrentMap.Delete(accessKey)
-		// 	}
-		// }
+		// iteration works!
+		var toDelete []string
+		for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+			found := false
+			for _, newTorrent := range newTorrents {
+				if newTorrent.ID == el.Value.AccessKey {
+					found = true
+					break
+				}
+			}
+			if !found {
+				toDelete = append(toDelete, el.Key)
+			}
+		}
+		for _, accessKey := range toDelete {
+			t.TorrentMap.Delete(accessKey)
+		}
 
 		wg.Wait()
 		close(torrentsChan)
@@ -243,9 +245,9 @@ func (t *TorrentManager) startRefreshJob() {
 		t.checksum = t.getChecksum()
 		t.mu.Unlock()
 
-		// if t.config.EnableRepair() {
-		// 	go t.repairAll()
-		// }
+		if t.config.EnableRepair() {
+			go t.repairAll()
+		}
 		go OnLibraryUpdateHook(t.config)
 	}
 }
@@ -289,34 +291,37 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		})
 	}
 	if selectedFiles.Len() > len(info.Links) && info.Progress == 100 {
-		t.log.Debugf("Some links has expired for %s %s: %d selected but only %d link(s)", info.ID, info.Name, selectedFiles.Len(), len(info.Links))
+		t.log.Debugf("%d links has expired for %s %s", selectedFiles.Len()-len(info.Links), info.ID, info.Name)
 		// chaotic file means RD will not output the desired file selection
 		// e.g. even if we select just a single mkv, it will output a rar
 		var isChaotic bool
 		selectedFiles, isChaotic = t.organizeChaos(&rdTorrent, selectedFiles)
-		if isChaotic && selectedFiles.Len() == 1 {
-			t.log.Infof("Torrent %s %s is unfixable, it's always returning an unstreamable link, ignoring", info.ID, info.Name)
+		if isChaotic {
+			t.log.Errorf("Torrent id=%s %s is unfixable, it's always returning an unstreamable link (it is no longer shown in your directories)", info.ID, info.Name)
 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+			return nil
 		} else {
 			if streamableCount > 1 {
 				// case for repair 1: it's missing some links (or all links)
 				// if we download it as is, we might get the same file over and over again
 				// so we need to redownload it with other files selected
 				// that is why we check if there are other streamable files
-				t.log.Infof("Torrent %s %s marked for repair", info.ID, info.Name)
+				t.log.Infof("Torrent id=%s %s marked for repair", info.ID, info.Name)
 				forRepair = true
 			} else {
-				t.log.Infof("Torrent %s %s is unfixable, the lone streamable link has expired, ignoring", info.ID, info.Name)
+				t.log.Errorf("Torrent id=%s %s is unfixable, the lone streamable link has expired (it is no longer shown in your directories)", info.ID, info.Name)
 				t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+				return nil
 			}
 		}
 	} else if selectedFiles.Len() == len(info.Links) {
 		// all links are still intact! good!
+		// iteration works!
 		i := 0
 		for el := selectedFiles.Front(); el != nil; el = el.Next() {
 			if i < len(info.Links) {
 				file := el.Value
-				file.Link = info.Links[i]
+				file.Link = info.Links[i] // verified working!
 				selectedFiles.Set(el.Key, file)
 				i++
 			}
@@ -446,11 +451,9 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 	}
 
 	go func() {
-		t.log.Debugf("Checking %d link(s) for problematic torrent id=%s", len(info.Links), info.ID)
 		wg.Wait()
 		close(sem)
 		close(resultsChan)
-		t.log.Debugf("Closing results channel for torrent id=%s, checking...", info.ID)
 	}()
 
 	isChaotic := false
@@ -459,9 +462,9 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 			continue
 		}
 		found := false
+		// iteration works!
 		for el := selectedFiles.Front(); el != nil; el = el.Next() {
 			if file, _ := selectedFiles.Get(el.Key); strings.Contains(file.Path, result.Response.Filename) {
-				t.log.Debugf("Found a file that is in the selection for torrent id=%s: %s", info.ID, result.Response.Filename)
 				file.Link = result.Response.Link
 				found = true
 			}
@@ -478,6 +481,8 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 					},
 					Link: result.Response.Link,
 				})
+			} else {
+				isChaotic = true
 			}
 		}
 	}
@@ -485,47 +490,48 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 	return selectedFiles, isChaotic
 }
 
-// func (t *TorrentManager) repairAll() {
-// 	for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
-// 		torrent := el.Value
-// 		// do not repair if: in progress
-// 		if torrent.InProgress {
-// 			continue
-// 		}
+func (t *TorrentManager) repairAll() {
+	// iteration works!
+	for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+		torrent := el.Value
+		// do not repair if in progress
+		if torrent.InProgress {
+			continue
+		}
 
-// 		var missingFiles []File
-// 		for el2 := torrent.SelectedFiles.Front(); el2 != nil; el2 = el2.Next() {
-// 			file, ok := torrent.SelectedFiles.Get(el2.Key)
-// 			if !ok {
-// 				continue
-// 			}
-// 			// check for case of repairs like
-// 			// case 1: missing links
-// 			// case 2: unrestrictable links TODO
-// 			if file.Link == "" {
-// 				missingFiles = append(missingFiles, *file)
-// 			}
-// 		}
-// 		if len(missingFiles) == 0 {
-// 			continue
-// 		}
+		var missingFiles []File
+		for el2 := torrent.SelectedFiles.Front(); el2 != nil; el2 = el2.Next() {
+			file, ok := torrent.SelectedFiles.Get(el2.Key)
+			if !ok {
+				continue
+			}
+			// check for case of repairs like
+			// case 1: missing links
+			// case 2: unrestrictable links TODO
+			if file.Link == "" {
+				missingFiles = append(missingFiles, *file)
+			}
+		}
+		if len(missingFiles) == 0 {
+			continue
+		}
 
-// 		for _, info := range torrent.Instances {
-// 			if info.ForRepair {
-// 				t.log.Infof("There were less links than was expected on %s %s; fixing...", info.ID, info.Name)
-// 				// t.repair(&info, true)
-// 				break // only repair the first one for repair and then move on
-// 			}
-// 			if len(info.Links) == 0 && info.Progress == 100 {
-// 				// If the torrent has no links
-// 				// and already processing repair
-// 				// delete it!
-// 				t.log.Infof("Deleting broken torrent id=%s as it doesn't contain any files", info.ID)
-// 				t.rd.DeleteTorrent(info.ID)
-// 			}
-// 		}
-// 	}
-// }
+		for _, info := range torrent.Instances {
+			if info.ForRepair {
+				t.log.Infof("There were less links than was expected on %s %s; fixing...", info.ID, info.Name)
+				// t.repair(&info, true)
+				break // only repair the first one for repair and then move on
+			}
+			if len(info.Links) == 0 && info.Progress == 100 {
+				// If the torrent has no links
+				// and already processing repair
+				// delete it!
+				t.log.Infof("Deleting broken torrent id=%s as it doesn't contain any files", info.ID)
+				t.rd.DeleteTorrent(info.ID)
+			}
+		}
+	}
+}
 
 // func (t *TorrentManager) repair(info *realdebrid.TorrentInfo, tryReinsertionFirst bool) {
 // 	// file.Link == "" should be repaired
@@ -583,7 +589,7 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 // 				t.log.Info("No other missing files left to reinsert")
 // 			}
 // 		} else {
-// 			t.log.Infof("Torrent id=%s is unfixable as the only link cached in RD is already broken", info.ID)
+// 			t.log.Infof("Torrent id=%s is unfixable as the only link cached in RD is already broken (it is no longer shown in your directories)", info.ID)
 // 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 // 			return
 // 		}
diff --git a/internal/universal/get.go b/internal/universal/get.go
index f699433..6f190d0 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -6,13 +6,12 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
-	"github.com/cenkalti/backoff"
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/dav"
 	intHttp "github.com/debridmediamanager.com/zurg/internal/http"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
+	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/hashicorp/golang-lru/v2/expirable"
 	"go.uber.org/zap"
@@ -73,6 +72,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 
 	resp := t.UnrestrictUntilOk(link)
 	if resp == nil {
+		log.Errorf("The link cannot be unrestricted, file %s is no longer available", file.Path)
 		// TODO: maybe repair the torrent?
 		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
 		return
@@ -100,55 +100,36 @@ func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, t
 		return
 	}
 
-	// Copy the headers from the incoming request to the new request.
-	for k, values := range r.Header {
-		for _, v := range values {
-			req.Header.Add(k, v)
-		}
+	// copy range header if it exists
+	if r.Header.Get("Range") != "" {
+		req.Header.Add("Range", r.Header.Get("Range"))
 	}
 
 	// Create a custom HTTP client with a timeout.
-	client := &http.Client{
-		Timeout: time.Second * 30, // Full request timeout including dial, request and response.
-	}
+	client := zurghttp.NewHTTPClient(c.GetToken(), 10)
 
-	// Define a retry policy with exponential backoff.
-	retryPolicy := backoff.NewExponentialBackOff()
-	retryPolicy.MaxElapsedTime = time.Minute * 2 // Set the maximum elapsed time for retries.
-
-	// Use an operation with retries.
-	operation := func() error {
-		resp, err := client.Do(req)
-		if err != nil {
-			log.Errorf("Error downloading file: %v", err)
-			return err
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode >= 500 {
-			log.Errorf("Received a 5XX status code: %d", resp.StatusCode)
-			return backoff.Permanent(err) // Stop retrying on bad status code.
-		}
-
-		// Copy the headers from the response to the ResponseWriter.
-		for k, vv := range resp.Header {
-			for _, v := range vv {
-				w.Header().Add(k, v)
-			}
-		}
-
-		// Stream the content to the ResponseWriter.
-		buf := make([]byte, c.GetNetworkBufferSize())
-		_, err = io.CopyBuffer(w, resp.Body, buf)
-		return err
-	}
-
-	// Perform the operation with the retry policy.
-	if err := backoff.Retry(operation, retryPolicy); err != nil {
-		log.Errorf("Failed after retries: %v", err)
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Errorf("Error downloading file %v", err)
 		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, t, c, log)
 		return
 	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
+		log.Errorf("Received a nonOK status code %d", resp.StatusCode)
+		streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, t, c, log)
+		return
+	}
+
+	for k, vv := range resp.Header {
+		for _, v := range vv {
+			w.Header().Add(k, v)
+		}
+	}
+
+	buf := make([]byte, c.GetNetworkBufferSize())
+	io.CopyBuffer(w, resp.Body, buf)
 }
 
 func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
diff --git a/internal/universal/util.go b/internal/universal/util.go
deleted file mode 100644
index 897888b..0000000
--- a/internal/universal/util.go
+++ /dev/null
@@ -1 +0,0 @@
-package universal
diff --git a/pkg/http/client.go b/pkg/http/client.go
index 50b65ad..9aae92f 100644
--- a/pkg/http/client.go
+++ b/pkg/http/client.go
@@ -29,10 +29,10 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 	return resp, err
 }
 
-func NewHTTPClient(token string, maxRetries int, timeout time.Duration) *HTTPClient {
+func NewHTTPClient(token string, maxRetries int) *HTTPClient {
 	return &HTTPClient{
 		BearerToken: token,
-		Client:      &http.Client{Timeout: timeout},
+		Client:      &http.Client{},
 		MaxRetries:  maxRetries,
 		Backoff: func(attempt int) time.Duration {
 			return time.Duration(attempt) * time.Second
diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go
index 6eecce7..a13fd78 100644
--- a/pkg/realdebrid/api.go
+++ b/pkg/realdebrid/api.go
@@ -9,7 +9,6 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
-	"time"
 
 	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
 	"go.uber.org/zap"
@@ -22,9 +21,7 @@ type RealDebrid struct {
 
 func NewRealDebrid(accessToken string, log *zap.SugaredLogger) *RealDebrid {
 	maxRetries := 10
-	timeout := 10 * time.Second
-	client := zurghttp.NewHTTPClient(accessToken, maxRetries, timeout)
-	log.Debugf("Created an HTTP client with %d max retries and %s timeout", maxRetries, timeout)
+	client := zurghttp.NewHTTPClient(accessToken, maxRetries)
 	return &RealDebrid{
 		log:    log,
 		client: client,

From 77487a6ac3d4b9e140c4e2cf50673d571210b104 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Fri, 10 Nov 2023 21:41:56 +0100
Subject: [PATCH 14/19] Remove destructive stuffs

---
 internal/torrent/manager.go | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index a7eabc8..2775279 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -470,7 +470,6 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 			}
 		}
 		if !found {
-			t.log.Debugf("Found a file that is NOT in the selection for torrent id=%s: %s %v", info.ID, result.Response.Filename, result.Response.Streamable)
 			if result.Response.Streamable == 1 {
 				selectedFiles.Set(filepath.Base(result.Response.Filename), &File{
 					File: realdebrid.File{
@@ -522,13 +521,6 @@ func (t *TorrentManager) repairAll() {
 				// t.repair(&info, true)
 				break // only repair the first one for repair and then move on
 			}
-			if len(info.Links) == 0 && info.Progress == 100 {
-				// If the torrent has no links
-				// and already processing repair
-				// delete it!
-				t.log.Infof("Deleting broken torrent id=%s as it doesn't contain any files", info.ID)
-				t.rd.DeleteTorrent(info.ID)
-			}
 		}
 	}
 }

From 147c0bd4443440c9f5f81e54abdccd5c2735fd7b Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Fri, 10 Nov 2023 23:35:33 +0100
Subject: [PATCH 15/19] Finalize all repair fixes

---
 internal/torrent/manager.go | 379 ++++++++++++++++++------------------
 internal/torrent/types.go   |   1 +
 internal/universal/get.go   |  34 ++--
 pkg/realdebrid/types.go     |   1 +
 4 files changed, 209 insertions(+), 206 deletions(-)

diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index 2775279..3715f3f 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -93,7 +93,7 @@ func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
 	merged := t1
 
 	// Merge SelectedFiles
-	// iteration works!
+	// side note: iteration works!
 	for el := t2.SelectedFiles.Front(); el != nil; el = el.Next() {
 		if _, ok := merged.SelectedFiles.Get(el.Key); !ok {
 			merged.SelectedFiles.Set(el.Key, el.Value)
@@ -112,7 +112,9 @@ func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
 	for _, instance := range merged.Instances {
 		if instance.Progress != 100 {
 			merged.InProgress = true
-			break
+		}
+		if instance.ForRepair {
+			merged.ForRepair = true
 		}
 	}
 
@@ -121,7 +123,10 @@ func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
 
 // proxy
 func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictResponse {
-	return t.rd.UnrestrictUntilOk(link)
+	t.workerPool <- true
+	ret := t.rd.UnrestrictUntilOk(link)
+	<-t.workerPool
+	return ret
 }
 
 type torrentsResponse struct {
@@ -211,7 +216,7 @@ func (t *TorrentManager) startRefreshJob() {
 			}(i)
 		}
 
-		// iteration works!
+		// side note: iteration works!
 		var toDelete []string
 		for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
 			found := false
@@ -295,9 +300,9 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		// chaotic file means RD will not output the desired file selection
 		// e.g. even if we select just a single mkv, it will output a rar
 		var isChaotic bool
-		selectedFiles, isChaotic = t.organizeChaos(&rdTorrent, selectedFiles)
+		selectedFiles, isChaotic = t.organizeChaos(info.Links, selectedFiles)
 		if isChaotic {
-			t.log.Errorf("Torrent id=%s %s is unfixable, it's always returning an unstreamable link (it is no longer shown in your directories)", info.ID, info.Name)
+			t.log.Errorf("Torrent id=%s %s is unfixable, it is always returning an unstreamable link (it is no longer shown in your directories)", info.ID, info.Name)
 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 			return nil
 		} else {
@@ -316,7 +321,7 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		}
 	} else if selectedFiles.Len() == len(info.Links) {
 		// all links are still intact! good!
-		// iteration works!
+		// side note: iteration works!
 		i := 0
 		for el := selectedFiles.Front(); el != nil; el = el.Next() {
 			if i < len(info.Links) {
@@ -428,31 +433,27 @@ func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo
 	return &torrent
 }
 
-func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *orderedmap.OrderedMap[string, *File]) (*orderedmap.OrderedMap[string, *File], bool) {
+func (t *TorrentManager) organizeChaos(links []string, selectedFiles *orderedmap.OrderedMap[string, *File]) (*orderedmap.OrderedMap[string, *File], bool) {
 	type Result struct {
 		Response *realdebrid.UnrestrictResponse
 	}
 
-	resultsChan := make(chan Result, len(info.Links))
+	resultsChan := make(chan Result, len(links))
 	var wg sync.WaitGroup
 
-	// Limit concurrency
-	sem := make(chan bool, t.config.GetNumOfWorkers())
-
-	for _, link := range info.Links {
+	for _, link := range links {
 		wg.Add(1)
-		sem <- true
 		go func(lnk string) {
 			defer wg.Done()
-			defer func() { <-sem }()
+			t.workerPool <- true
 			resp := t.rd.UnrestrictUntilOk(lnk)
+			<-t.workerPool
 			resultsChan <- Result{Response: resp}
 		}(link)
 	}
 
 	go func() {
 		wg.Wait()
-		close(sem)
 		close(resultsChan)
 	}()
 
@@ -462,7 +463,7 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 			continue
 		}
 		found := false
-		// iteration works!
+		// side note: iteration works!
 		for el := selectedFiles.Front(); el != nil; el = el.Next() {
 			if file, _ := selectedFiles.Get(el.Key); strings.Contains(file.Path, result.Response.Filename) {
 				file.Link = result.Response.Link
@@ -490,7 +491,7 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles *
 }
 
 func (t *TorrentManager) repairAll() {
-	// iteration works!
+	// side note: iteration works!
 	for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
 		torrent := el.Value
 		// do not repair if in progress
@@ -498,199 +499,191 @@ func (t *TorrentManager) repairAll() {
 			continue
 		}
 
-		var missingFiles []File
+		// do not repair if all files have links
+		forRepair := false
 		for el2 := torrent.SelectedFiles.Front(); el2 != nil; el2 = el2.Next() {
-			file, ok := torrent.SelectedFiles.Get(el2.Key)
-			if !ok {
-				continue
-			}
-			// check for case of repairs like
-			// case 1: missing links
-			// case 2: unrestrictable links TODO
+			file := el2.Value
 			if file.Link == "" {
-				missingFiles = append(missingFiles, *file)
+				forRepair = true
+				break
 			}
 		}
-		if len(missingFiles) == 0 {
+		if !forRepair {
+			// if it was marked for repair, unmark it
+			torrent.ForRepair = false
+			t.TorrentMap.Set(torrent.AccessKey, torrent)
 			continue
 		}
 
-		for _, info := range torrent.Instances {
-			if info.ForRepair {
-				t.log.Infof("There were less links than was expected on %s %s; fixing...", info.ID, info.Name)
-				// t.repair(&info, true)
-				break // only repair the first one for repair and then move on
-			}
+		// when getting info, we mark it for repair if it's missing some links
+		if torrent.ForRepair {
+			t.log.Infof("There were less links than was expected on %s; fixing...", torrent.AccessKey)
+			t.Repair(torrent.AccessKey)
+			break // only repair the first one for repair and then move on
 		}
 	}
 }
 
-// func (t *TorrentManager) repair(info *realdebrid.TorrentInfo, tryReinsertionFirst bool) {
-// 	// file.Link == "" should be repaired
-// 	// then we repair it!
-// 	t.log.Infof("Repairing torrent id=%s", info.ID)
-// 	// check if we can still add more downloads
-// 	proceed := t.canCapacityHandle()
-// 	if !proceed {
-// 		t.log.Error("Cannot add more torrents, exiting")
-// 		return
-// 	}
+func (t *TorrentManager) Repair(accessKey string) {
+	torrent, _ := t.TorrentMap.Get(accessKey)
+	if torrent == nil {
+		t.log.Errorf("Cannot find torrent %s anymore to repair it", accessKey)
+		return
+	}
+	if torrent.InProgress {
+		t.log.Infof("Torrent %s is in progress, cannot repair", torrent.AccessKey)
+		return
+	}
 
-// 	// first solution: add the same selection, maybe it can be fixed by reinsertion?
-// 	success := false
-// 	if tryReinsertionFirst {
-// 		success = t.reinsertTorrent(info, "", true)
-// 	}
-// 	if !success {
-// 		// if all the selected files are missing but there are other streamable files
-// 		var otherStreamableFileIDs []int
-// 		for _, file := range info.Files {
-// 			found := false
-// 			for el := selectedFiles.Front(); el != nil; el = el.Next() {
+	// check if we can still add more downloads
+	proceed := t.canCapacityHandle()
+	if !proceed {
+		t.log.Error("Cannot add more torrents, ignoring repair request")
+		return
+	}
 
-// 			}
-// 			for _, selectedFile := range selectedFiles {
-// 				if selectedFile.ID == file.ID {
-// 					found = true
-// 					break
-// 				}
-// 			}
-// 			if !found && isStreamable(file.Path) {
-// 				otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID)
-// 			}
-// 		}
-// 		if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 {
-// 			// we will download 1 extra streamable file to force a redownload of the missing files
-// 			// or if there's only 1 missing file, we will download 1 more to prevent a rename
-// 			missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",")
-// 			t.log.Infof("Redownloading %d missing files", len(missingFiles))
-// 			t.reinsertTorrent(info, missingFilesPlus1, false)
-// 		} else if len(selectedFiles) > 1 {
-// 			// if not, last resort: add only the missing files but do it in 2 batches
-// 			half := len(missingFiles) / 2
-// 			missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
-// 			missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
-// 			if missingFiles1 != "" {
-// 				t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1))
-// 				t.reinsertTorrent(info, missingFiles1, false)
-// 			}
-// 			if missingFiles2 != "" {
-// 				t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2))
-// 				t.reinsertTorrent(info, missingFiles2, false)
-// 			} else {
-// 				t.log.Info("No other missing files left to reinsert")
-// 			}
-// 		} else {
-// 			t.log.Infof("Torrent id=%s is unfixable as the only link cached in RD is already broken (it is no longer shown in your directories)", info.ID)
-// 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
-// 			return
-// 		}
-// 		t.log.Info("Waiting for downloads to finish")
-// 	}
-// }
+	// make the file messy
+	var links []string
+	for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+		file := el.Value
+		if file.Link != "" {
+			links = append(links, file.Link)
+		}
+		file.Link = ""
+	}
+	selectedFiles, _ := t.organizeChaos(links, torrent.SelectedFiles)
+	torrent.SelectedFiles = selectedFiles
+	t.TorrentMap.Set(torrent.AccessKey, torrent)
 
-// func (t *TorrentManager) reinsertTorrent(torrent *realdebrid.TorrentInfo, missingFiles string, deleteIfFailed bool) bool {
-// 	// if missingFiles is not provided, look for missing files
-// 	if missingFiles == "" {
-// 		var tmpSelection string
-// 		for _, file := range torrent.Files {
-// 			if file.Selected == 0 {
-// 				continue
-// 			}
-// 			tmpSelection += fmt.Sprintf("%d,", file.ID)
-// 		}
-// 		if tmpSelection == "" {
-// 			return false
-// 		}
-// 		if len(tmpSelection) > 0 {
-// 			missingFiles = tmpSelection[:len(tmpSelection)-1]
-// 		}
-// 	}
+	// first solution: add the same selection, maybe it can be fixed by reinsertion?
+	if t.reinsertTorrent(torrent, "") {
+		t.log.Infof("Redownloaded torrent %s to repair it", torrent.AccessKey)
+		return
+	}
+	// if all the selected files are missing but there are other streamable files
+	var missingFiles []File
+	for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+		file := el.Value
+		if file.Link == "" {
+			missingFiles = append(missingFiles, *file)
+		}
+	}
+	if len(missingFiles) > 0 {
+		t.log.Infof("Redownloading %d missing files for torrent %s", len(missingFiles), torrent.AccessKey)
+		// if not, last resort: add only the missing files but do it in 2 batches
+		half := len(missingFiles) / 2
+		missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
+		missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
+		if missingFiles1 != "" {
+			t.reinsertTorrent(torrent, missingFiles1)
+		}
+		if missingFiles2 != "" {
+			t.reinsertTorrent(torrent, missingFiles2)
+		}
+	}
+}
 
-// 	// redownload torrent
-// 	resp, err := t.rd.AddMagnetHash(torrent.Hash)
-// 	if err != nil {
-// 		t.log.Errorf("Cannot redownload torrent: %v", err)
-// 		return false
-// 	}
-// 	newTorrentID := resp.ID
-// 	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
-// 	if err != nil {
-// 		t.log.Errorf("Cannot start redownloading: %v", err)
-// 	}
+func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string) bool {
+	// if missingFiles is not provided, look for missing files
+	if missingFiles == "" {
+		var tmpSelection string
+		for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+			file := el.Value
+			tmpSelection += fmt.Sprintf("%d,", file.ID)
+		}
+		if tmpSelection == "" {
+			return false
+		}
+		if len(tmpSelection) > 0 {
+			missingFiles = tmpSelection[:len(tmpSelection)-1]
+		}
+	}
 
-// 	if deleteIfFailed {
-// 		if err != nil {
-// 			t.rd.DeleteTorrent(newTorrentID)
-// 			return false
-// 		}
-// 		time.Sleep(1 * time.Second)
-// 		// see if the torrent is ready
-// 		info, err := t.rd.GetTorrentInfo(newTorrentID)
-// 		if err != nil {
-// 			t.log.Errorf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
-// 			if deleteIfFailed {
-// 				t.rd.DeleteTorrent(newTorrentID)
-// 			}
-// 			return false
-// 		}
-// 		time.Sleep(1 * time.Second)
+	// redownload torrent
+	resp, err := t.rd.AddMagnetHash(torrent.Instances[0].Hash)
+	if err != nil {
+		t.log.Errorf("Cannot redownload torrent: %v", err)
+		return false
+	}
+	time.Sleep(1 * time.Second)
 
-// 		if info.Progress != 100 {
-// 			t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion, currently %d%%", info.ID, info.Progress)
-// 			t.rd.DeleteTorrent(newTorrentID)
-// 			return false
-// 		}
+	// select files
+	newTorrentID := resp.ID
+	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
+	if err != nil {
+		t.log.Errorf("Cannot start redownloading: %v", err)
+		t.rd.DeleteTorrent(newTorrentID)
+		return false
+	}
+	time.Sleep(10 * time.Second)
 
-// 		missingCount := len(strings.Split(missingFiles, ","))
-// 		if len(info.Links) != missingCount {
-// 			t.log.Infof("It didn't fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount)
-// 			t.rd.DeleteTorrent(newTorrentID)
-// 			return false
-// 		}
-// 		t.log.Infof("Redownload successful id=%s, deleting old torrent id=%s", newTorrentID, torrent.ID)
-// 		t.rd.DeleteTorrent(torrent.ID)
-// 	}
-// 	return true
-// }
+	// see if the torrent is ready
+	info, err := t.rd.GetTorrentInfo(newTorrentID)
+	if err != nil {
+		t.log.Errorf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
+		t.rd.DeleteTorrent(newTorrentID)
+		return false
+	}
 
-// func (t *TorrentManager) canCapacityHandle() bool {
-// 	// max waiting time is 45 minutes
-// 	const maxRetries = 50
-// 	const baseDelay = 1 * time.Second
-// 	const maxDelay = 60 * time.Second
-// 	retryCount := 0
-// 	for {
-// 		count, err := t.rd.GetActiveTorrentCount()
-// 		if err != nil {
-// 			t.log.Errorf("Cannot get active downloads count: %v", err)
-// 			if retryCount >= maxRetries {
-// 				t.log.Error("Max retries reached. Exiting.")
-// 				return false
-// 			}
-// 			delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
-// 			if delay > maxDelay {
-// 				delay = maxDelay
-// 			}
-// 			time.Sleep(delay)
-// 			retryCount++
-// 			continue
-// 		}
+	if info.Status == "magnet_error" || info.Status == "error" || info.Status == "virus" || info.Status == "dead" {
+		t.log.Errorf("Redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status)
+		t.rd.DeleteTorrent(newTorrentID)
+		return false
+	}
 
-// 		if count.DownloadingCount < count.MaxNumberOfTorrents {
-// 			t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount)
-// 			return true
-// 		}
+	if info.Progress != 100 {
+		t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion (this should fix the issue already)", info.ID)
+		return true
+	}
 
-// 		if retryCount >= maxRetries {
-// 			t.log.Error("Max retries reached, exiting")
-// 			return false
-// 		}
-// 		delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
-// 		if delay > maxDelay {
-// 			delay = maxDelay
-// 		}
-// 		time.Sleep(delay)
-// 		retryCount++
-// 	}
-// }
+	missingCount := len(strings.Split(missingFiles, ","))
+	if len(info.Links) != missingCount {
+		t.log.Infof("It did not fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount)
+		t.rd.DeleteTorrent(newTorrentID)
+		return false
+	}
+
+	t.log.Infof("Repair successful id=%s", newTorrentID)
+	return true
+}
+
+func (t *TorrentManager) canCapacityHandle() bool {
+	// max waiting time is 45 minutes
+	const maxRetries = 50
+	const baseDelay = 1 * time.Second
+	const maxDelay = 60 * time.Second
+	retryCount := 0
+	for {
+		count, err := t.rd.GetActiveTorrentCount()
+		if err != nil {
+			t.log.Errorf("Cannot get active downloads count: %v", err)
+			if retryCount >= maxRetries {
+				t.log.Error("Max retries reached. Exiting.")
+				return false
+			}
+			delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+			if delay > maxDelay {
+				delay = maxDelay
+			}
+			time.Sleep(delay)
+			retryCount++
+			continue
+		}
+
+		if count.DownloadingCount < count.MaxNumberOfTorrents {
+			t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount)
+			return true
+		}
+
+		if retryCount >= maxRetries {
+			t.log.Error("Max retries reached, exiting")
+			return false
+		}
+		delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+		if delay > maxDelay {
+			delay = maxDelay
+		}
+		time.Sleep(delay)
+		retryCount++
+	}
+}
diff --git a/internal/torrent/types.go b/internal/torrent/types.go
index f4c35a7..9e1888f 100644
--- a/internal/torrent/types.go
+++ b/internal/torrent/types.go
@@ -11,6 +11,7 @@ type Torrent struct {
 	Directories   []string
 	LatestAdded   string
 	InProgress    bool
+	ForRepair     bool
 
 	Instances []realdebrid.TorrentInfo
 }
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 6f190d0..4551039 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -40,10 +40,6 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		}
 		return
 	}
-	if data, exists := cache.Get(requestPath); exists {
-		streamFileToResponse(data, w, r, t, c, log)
-		return
-	}
 
 	baseDirectory := segments[len(segments)-3]
 	accessKey := segments[len(segments)-2]
@@ -62,9 +58,15 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
+
+	if data, exists := cache.Get(requestPath); exists {
+		streamFileToResponse(torrent, data, w, r, t, c, log)
+		return
+	}
+
 	if file.Link == "" {
 		// This is a dead file, serve an alternate file
-		log.Errorf("File %s is no longer available", filename)
+		log.Errorf("File %s is not yet available, zurg is repairing the torrent", filename)
 		streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, t, c, log)
 		return
 	}
@@ -72,8 +74,8 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 
 	resp := t.UnrestrictUntilOk(link)
 	if resp == nil {
-		log.Errorf("The link cannot be unrestricted, file %s is no longer available", file.Path)
-		// TODO: maybe repair the torrent?
+		go t.Repair(torrent.AccessKey)
+		log.Errorf("File %s is no longer available, torrent is marked for repair", file.Path)
 		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
 		return
 	} else if resp.Filename != filename {
@@ -88,10 +90,10 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		}
 	}
 	cache.Add(requestPath, resp.Download)
-	streamFileToResponse(resp.Download, w, r, t, c, log)
+	streamFileToResponse(torrent, resp.Download, w, r, t, c, log)
 }
 
-func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
+func streamFileToResponse(torrent *torrent.Torrent, url string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
 	// Create a new request for the file download.
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
@@ -105,19 +107,25 @@ func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, t
 		req.Header.Add("Range", r.Header.Get("Range"))
 	}
 
-	// Create a custom HTTP client with a timeout.
+	// Create a custom HTTP client
 	client := zurghttp.NewHTTPClient(c.GetToken(), 10)
 
 	resp, err := client.Do(req)
 	if err != nil {
-		log.Errorf("Error downloading file %v", err)
+		log.Errorf("Error downloading file %v ; torrent is marked for repair", err)
+		if torrent != nil {
+			go t.Repair(torrent.AccessKey)
+		}
 		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, t, c, log)
 		return
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		log.Errorf("Received a nonOK status code %d", resp.StatusCode)
+		log.Errorf("Received a %s status code ; torrent is marked for repair", resp.Status)
+		if torrent != nil {
+			go t.Repair(torrent.AccessKey)
+		}
 		streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, t, c, log)
 		return
 	}
@@ -138,5 +146,5 @@ func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, t *to
 		http.Error(w, "REAL-DEBRID IS DOWN", http.StatusInternalServerError)
 		return
 	}
-	streamFileToResponse(resp.Download, w, r, t, c, log)
+	streamFileToResponse(nil, resp.Download, w, r, t, c, log)
 }
diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go
index 230cf19..a27fb92 100644
--- a/pkg/realdebrid/types.go
+++ b/pkg/realdebrid/types.go
@@ -34,6 +34,7 @@ type TorrentInfo struct {
 	Name          string   `json:"filename"`
 	Hash          string   `json:"hash"`
 	Progress      int      `json:"-"`
+	Status        string   `json:"status"`
 	Added         string   `json:"added"`
 	Bytes         int64    `json:"bytes"`
 	Links         []string `json:"links"`

From cd96c7bd38d9a0685b617627bb539177bef4c8a3 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Sat, 11 Nov 2023 02:34:46 +0100
Subject: [PATCH 16/19] fixes here and there

---
 cmd/zurg/main.go            | 18 +++------
 go.mod                      | 15 +-------
 go.sum                      | 29 --------------
 hosts.txt                   | 76 +++++++++++++++++++++++++++++++++++++
 internal/config/load.go     |  1 +
 internal/config/types.go    | 35 +++++++++++------
 internal/dav/listing.go     |  8 ++--
 internal/http/listing.go    |  4 +-
 internal/torrent/manager.go | 47 +++++++++++++----------
 internal/universal/get.go   | 18 ++++-----
 internal/universal/head.go  |  6 +--
 pkg/dav/response.go         |  4 +-
 pkg/http/client.go          | 27 +++++++++++--
 pkg/realdebrid/api.go       | 48 +++--------------------
 14 files changed, 181 insertions(+), 155 deletions(-)
 create mode 100644 hosts.txt

diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index 392d7f6..f63d6e5 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -13,31 +13,23 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/net"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
+	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/hashicorp/golang-lru/v2/expirable"
-	"github.com/nutsdb/nutsdb"
 )
 
 func main() {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("zurg")
+	log := logutil.NewLogger().Named("zurg")
 
 	config, configErr := config.LoadZurgConfig("./config.yml")
 	if configErr != nil {
 		log.Panicf("Config failed to load: %v", configErr)
 	}
 
-	db, err := nutsdb.Open(
-		nutsdb.DefaultOptions,
-		nutsdb.WithDir("/tmp/nutsdb"),
-	)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer db.Close()
-
 	cache := expirable.NewLRU[string, string](1e4, nil, time.Hour)
 
-	torrentMgr := torrent.NewTorrentManager(config, db)
+	rd := realdebrid.NewRealDebrid(config.GetToken(), config, logutil.NewLogger().Named("realdebrid"))
+
+	torrentMgr := torrent.NewTorrentManager(config, rd)
 
 	mux := http.NewServeMux()
 	net.Router(mux, config, torrentMgr, cache)
diff --git a/go.mod b/go.mod
index e5034e3..a7bb848 100644
--- a/go.mod
+++ b/go.mod
@@ -5,21 +5,8 @@ go 1.21.3
 require (
 	github.com/elliotchance/orderedmap/v2 v2.2.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/nutsdb/nutsdb v0.14.1
 	go.uber.org/zap v1.26.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require (
-	github.com/antlabs/stl v0.0.1 // indirect
-	github.com/antlabs/timer v0.0.11 // indirect
-	github.com/bwmarrin/snowflake v0.3.0 // indirect
-	github.com/emirpasic/gods v1.18.1 // indirect
-	github.com/gofrs/flock v0.8.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/tidwall/btree v1.6.0 // indirect
-	github.com/xujiajun/mmap-go v1.0.1 // indirect
-	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
-)
+require go.uber.org/multierr v1.10.0 // indirect
diff --git a/go.sum b/go.sum
index ff76c95..de0317f 100644
--- a/go.sum
+++ b/go.sum
@@ -1,49 +1,20 @@
-github.com/antlabs/stl v0.0.1 h1:TRD3csCrjREeLhLoQ/supaoCvFhNLBTNIwuRGrDIs6Q=
-github.com/antlabs/stl v0.0.1/go.mod h1:wvVwP1loadLG3cRjxUxK8RL4Co5xujGaZlhbztmUEqQ=
-github.com/antlabs/timer v0.0.11 h1:z75oGFLeTqJHMOcWzUPBKsBbQAz4Ske3AfqJ7bsdcwU=
-github.com/antlabs/timer v0.0.11/go.mod h1:JNV8J3yGvMKhCavGXgj9HXrVZkfdQyKCcqXBT8RdyuU=
-github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
-github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
 github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
-github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
-github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
-github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-github.com/nutsdb/nutsdb v0.14.1 h1:z+Kth/kz2oYqKmOMBZho1YK2183xjrcl6KExRtCFl18=
-github.com/nutsdb/nutsdb v0.14.1/go.mod h1:6inOji9rFBporXeHDjJny4g50RpQbkjSK5jI1hht0j8=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
-github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/xujiajun/mmap-go v1.0.1 h1:7Se7ss1fLPPRW+ePgqGpCkfGIZzJV6JPq9Wq9iv/WHc=
-github.com/xujiajun/mmap-go v1.0.1/go.mod h1:CNN6Sw4SL69Sui00p0zEzcZKbt+5HtEnYUsc6BKKRMg=
-github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 h1:w0si+uee0iAaCJO9q86T6yrhdadgcsoNuh47LrUykzg=
-github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235/go.mod h1:MR4+0R6A9NS5IABnIM3384FfOq8QFVnm7WDrBOhIaMU=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
 go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
-golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
-gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/hosts.txt b/hosts.txt
new file mode 100644
index 0000000..cd5a887
--- /dev/null
+++ b/hosts.txt
@@ -0,0 +1,76 @@
+20.download.real-debrid.cloud
+21.download.real-debrid.cloud
+22.download.real-debrid.cloud
+23.download.real-debrid.cloud
+30.download.real-debrid.cloud
+31.download.real-debrid.cloud
+32.download.real-debrid.cloud
+34.download.real-debrid.cloud
+40.download.real-debrid.cloud
+41.download.real-debrid.cloud
+42.download.real-debrid.cloud
+43.download.real-debrid.cloud
+44.download.real-debrid.cloud
+45.download.real-debrid.cloud
+50.download.real-debrid.cloud
+51.download.real-debrid.cloud
+52.download.real-debrid.cloud
+53.download.real-debrid.cloud
+54.download.real-debrid.cloud
+55.download.real-debrid.cloud
+56.download.real-debrid.cloud
+57.download.real-debrid.cloud
+58.download.real-debrid.cloud
+59.download.real-debrid.cloud
+60.download.real-debrid.cloud
+61.download.real-debrid.cloud
+62.download.real-debrid.cloud
+63.download.real-debrid.cloud
+64.download.real-debrid.cloud
+65.download.real-debrid.cloud
+66.download.real-debrid.cloud
+67.download.real-debrid.cloud
+68.download.real-debrid.cloud
+69.download.real-debrid.cloud
+20.download.real-debrid.com
+21.download.real-debrid.com
+22.download.real-debrid.com
+23.download.real-debrid.com
+30.download.real-debrid.com
+31.download.real-debrid.com
+32.download.real-debrid.com
+34.download.real-debrid.com
+40.download.real-debrid.com
+41.download.real-debrid.com
+42.download.real-debrid.com
+43.download.real-debrid.com
+44.download.real-debrid.com
+45.download.real-debrid.com
+50.download.real-debrid.com
+51.download.real-debrid.com
+52.download.real-debrid.com
+53.download.real-debrid.com
+54.download.real-debrid.com
+55.download.real-debrid.com
+56.download.real-debrid.com
+57.download.real-debrid.com
+58.download.real-debrid.com
+59.download.real-debrid.com
+60.download.real-debrid.com
+61.download.real-debrid.com
+62.download.real-debrid.com
+63.download.real-debrid.com
+64.download.real-debrid.com
+65.download.real-debrid.com
+66.download.real-debrid.com
+67.download.real-debrid.com
+68.download.real-debrid.com
+69.download.real-debrid.com
+hkg1.download.real-debrid.com
+lax1.download.real-debrid.com
+lon1.download.real-debrid.com
+mum1.download.real-debrid.com
+rbx.download.real-debrid.com
+sgp1.download.real-debrid.com
+tlv1.download.real-debrid.com
+tyo1.download.real-debrid.com
diff --git a/internal/config/load.go b/internal/config/load.go
index 1283c1a..b1bb3ba 100644
--- a/internal/config/load.go
+++ b/internal/config/load.go
@@ -23,6 +23,7 @@ type ConfigInterface interface {
 	GetNetworkBufferSize() int
 	GetMountPoint() string
 	EnableRetainFolderNameExtension() bool
+	GetRandomPreferredHost() string
 }
 
 func LoadZurgConfig(filename string) (ConfigInterface, error) {
diff --git a/internal/config/types.go b/internal/config/types.go
index ddca6a5..109554c 100644
--- a/internal/config/types.go
+++ b/internal/config/types.go
@@ -1,18 +1,21 @@
 package config
 
+import "math/rand"
+
 type ZurgConfig struct {
-	Version                   string `yaml:"zurg"`
-	Token                     string `yaml:"token"`
-	Host                      string `yaml:"host"`
-	Port                      string `yaml:"port"`
-	NumOfWorkers              int    `yaml:"concurrent_workers"`
-	RefreshEverySeconds       int    `yaml:"check_for_changes_every_secs"`
-	CacheTimeHours            int    `yaml:"info_cache_time_hours"`
-	CanRepair                 bool   `yaml:"enable_repair"`
-	OnLibraryUpdate           string `yaml:"on_library_update"`
-	NetworkBufferSize         int    `yaml:"network_buffer_size"`
-	MountPoint                string `yaml:"mount_point"`
-	RetainFolderNameExtension bool   `yaml:"retain_folder_name_extension"`
+	Version                   string   `yaml:"zurg"`
+	Token                     string   `yaml:"token"`
+	Host                      string   `yaml:"host"`
+	Port                      string   `yaml:"port"`
+	NumOfWorkers              int      `yaml:"concurrent_workers"`
+	RefreshEverySeconds       int      `yaml:"check_for_changes_every_secs"`
+	CacheTimeHours            int      `yaml:"info_cache_time_hours"`
+	CanRepair                 bool     `yaml:"enable_repair"`
+	OnLibraryUpdate           string   `yaml:"on_library_update"`
+	NetworkBufferSize         int      `yaml:"network_buffer_size"`
+	MountPoint                string   `yaml:"mount_point"`
+	RetainFolderNameExtension bool     `yaml:"retain_folder_name_extension"`
+	PreferredHosts            []string `yaml:"preferred_hosts"`
 }
 
 func (z *ZurgConfig) GetToken() string {
@@ -73,3 +76,11 @@ func (z *ZurgConfig) GetMountPoint() string {
 func (z *ZurgConfig) EnableRetainFolderNameExtension() bool {
 	return z.RetainFolderNameExtension
 }
+
+func (z *ZurgConfig) GetRandomPreferredHost() string {
+	if len(z.PreferredHosts) == 0 {
+		return ""
+	}
+	randomIndex := rand.Intn(len(z.PreferredHosts))
+	return z.PreferredHosts[randomIndex]
+}
diff --git a/internal/dav/listing.go b/internal/dav/listing.go
index 69bbca9..0b7bcf9 100644
--- a/internal/dav/listing.go
+++ b/internal/dav/listing.go
@@ -33,7 +33,7 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 	case len(filteredSegments) == 2:
 		output, err = handleSingleTorrent(requestPath, w, r, t)
 	default:
-		log.Errorf("Request %s %s not found", r.Method, requestPath)
+		log.Warnf("Request %s %s not found", r.Method, requestPath)
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
@@ -54,9 +54,9 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 
 func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface) ([]byte, error) {
 	var responses []dav.Response
-	responses = append(responses, dav.Directory("/"))
+	responses = append(responses, dav.Directory(""))
 	for _, directory := range c.GetDirectories() {
-		responses = append(responses, dav.Directory("/"+directory))
+		responses = append(responses, dav.Directory(directory))
 	}
 	rootResponse := dav.MultiStatus{
 		XMLNS:    "DAV:",
@@ -104,7 +104,7 @@ func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Requ
 	accessKey := path.Base(requestPath)
 	torrent, exists := t.TorrentMap.Get(accessKey)
 	if !exists {
-		return nil, fmt.Errorf("cannot find torrent %s", requestPath)
+		return nil, fmt.Errorf("cannot find torrent %s", accessKey)
 	}
 
 	var responses []dav.Response
diff --git a/internal/http/listing.go b/internal/http/listing.go
index 096c8f8..af7e399 100644
--- a/internal/http/listing.go
+++ b/internal/http/listing.go
@@ -30,7 +30,7 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
 	case len(filteredSegments) == 3:
 		output, err = handleSingleTorrent(requestPath, w, r, t)
 	default:
-		log.Errorf("Request %s %s not found", r.Method, requestPath)
+		log.Warnf("Request %s %s not found", r.Method, requestPath)
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
@@ -89,7 +89,7 @@ func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Requ
 	accessKey := path.Base(requestPath)
 	torrent, _ := t.TorrentMap.Get(accessKey)
 	if torrent == nil {
-		return nil, fmt.Errorf("cannot find torrent %s", requestPath)
+		return nil, fmt.Errorf("cannot find torrent %s", accessKey)
 	}
 
 	htmlDoc := "<ol>"
diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index 3715f3f..75c01c7 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -14,17 +14,16 @@ import (
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/elliotchance/orderedmap/v2"
-	"github.com/nutsdb/nutsdb"
 	"go.uber.org/zap"
 )
 
 type TorrentManager struct {
 	TorrentMap      *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
+	repairMap       *orderedmap.OrderedMap[string, bool]     // accessKey -> bool
 	requiredVersion string
 	rd              *realdebrid.RealDebrid
 	checksum        string
 	config          config.ConfigInterface
-	db              *nutsdb.DB
 	workerPool      chan bool
 	mu              *sync.Mutex
 	log             *zap.SugaredLogger
@@ -33,19 +32,18 @@ type TorrentManager struct {
 // NewTorrentManager creates a new torrent manager
 // it will fetch all torrents and their info in the background
 // and store them in-memory and cached in files
-func NewTorrentManager(config config.ConfigInterface, db *nutsdb.DB) *TorrentManager {
+func NewTorrentManager(config config.ConfigInterface, rd *realdebrid.RealDebrid) *TorrentManager {
 	t := &TorrentManager{
 		TorrentMap:      orderedmap.NewOrderedMap[string, *Torrent](),
+		repairMap:       orderedmap.NewOrderedMap[string, bool](),
 		requiredVersion: "10.11.2023",
-		rd:              realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
+		rd:              rd,
 		config:          config,
-		db:              db,
 		workerPool:      make(chan bool, config.GetNumOfWorkers()),
 		mu:              &sync.Mutex{},
 		log:             logutil.NewLogger().Named("manager"),
 	}
 
-	// start with a clean slate
 	t.mu.Lock()
 
 	newTorrents, _, err := t.rd.GetTorrents(0)
@@ -170,7 +168,7 @@ func (t *TorrentManager) getChecksum() string {
 			totalCount = torrentsResp.totalCount
 		case count = <-countChan:
 		case err := <-errChan:
-			t.log.Errorf("Checksum API Error: %v\n", err)
+			t.log.Warnf("Checksum API Error: %v\n", err)
 			return ""
 		}
 	}
@@ -199,7 +197,7 @@ func (t *TorrentManager) startRefreshJob() {
 
 		newTorrents, _, err := t.rd.GetTorrents(0)
 		if err != nil {
-			t.log.Errorf("Cannot get torrents: %v\n", err)
+			t.log.Warnf("Cannot get torrents: %v\n", err)
 			continue
 		}
 		t.log.Infof("Detected changes! Refreshing %d torrents", len(newTorrents))
@@ -271,7 +269,7 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 	if info == nil {
 		info, err = t.rd.GetTorrentInfo(rdTorrent.ID)
 		if err != nil {
-			t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
+			t.log.Warnf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
 			return nil
 		}
 	}
@@ -296,14 +294,13 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		})
 	}
 	if selectedFiles.Len() > len(info.Links) && info.Progress == 100 {
-		t.log.Debugf("%d links has expired for %s %s", selectedFiles.Len()-len(info.Links), info.ID, info.Name)
 		// chaotic file means RD will not output the desired file selection
 		// e.g. even if we select just a single mkv, it will output a rar
 		var isChaotic bool
 		selectedFiles, isChaotic = t.organizeChaos(info.Links, selectedFiles)
 		if isChaotic {
-			t.log.Errorf("Torrent id=%s %s is unfixable, it is always returning an unstreamable link (it is no longer shown in your directories)", info.ID, info.Name)
-			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+			// t.log.Warnf("Torrent id=%s %s is unfixable, it is always returning an unstreamable link (it is no longer shown in your directories)", info.ID, info.Name)
+			// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 			return nil
 		} else {
 			if streamableCount > 1 {
@@ -314,8 +311,8 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 				t.log.Infof("Torrent id=%s %s marked for repair", info.ID, info.Name)
 				forRepair = true
 			} else {
-				t.log.Errorf("Torrent id=%s %s is unfixable, the lone streamable link has expired (it is no longer shown in your directories)", info.ID, info.Name)
-				t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+				// t.log.Warnf("Torrent id=%s %s is unfixable, the lone streamable link has expired (it is no longer shown in your directories)", info.ID, info.Name)
+				// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 				return nil
 			}
 		}
@@ -525,9 +522,19 @@ func (t *TorrentManager) repairAll() {
 }
 
 func (t *TorrentManager) Repair(accessKey string) {
+	if _, exists := t.repairMap.Get(accessKey); exists {
+		return
+	}
+	t.repairMap.Set(accessKey, true)
+
+	if !t.config.EnableRepair() {
+		t.log.Warn("Repair is disabled; if you do not have other zurg instances running, you should enable repair")
+		return
+	}
+
 	torrent, _ := t.TorrentMap.Get(accessKey)
 	if torrent == nil {
-		t.log.Errorf("Cannot find torrent %s anymore to repair it", accessKey)
+		t.log.Warnf("Cannot find torrent %s anymore to repair it", accessKey)
 		return
 	}
 	if torrent.InProgress {
@@ -602,7 +609,7 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string)
 	// redownload torrent
 	resp, err := t.rd.AddMagnetHash(torrent.Instances[0].Hash)
 	if err != nil {
-		t.log.Errorf("Cannot redownload torrent: %v", err)
+		t.log.Warnf("Cannot redownload torrent: %v", err)
 		return false
 	}
 	time.Sleep(1 * time.Second)
@@ -611,7 +618,7 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string)
 	newTorrentID := resp.ID
 	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
 	if err != nil {
-		t.log.Errorf("Cannot start redownloading: %v", err)
+		t.log.Warnf("Cannot start redownloading: %v", err)
 		t.rd.DeleteTorrent(newTorrentID)
 		return false
 	}
@@ -620,13 +627,13 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string)
 	// see if the torrent is ready
 	info, err := t.rd.GetTorrentInfo(newTorrentID)
 	if err != nil {
-		t.log.Errorf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
+		t.log.Warnf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
 		t.rd.DeleteTorrent(newTorrentID)
 		return false
 	}
 
 	if info.Status == "magnet_error" || info.Status == "error" || info.Status == "virus" || info.Status == "dead" {
-		t.log.Errorf("Redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status)
+		t.log.Warnf("Redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status)
 		t.rd.DeleteTorrent(newTorrentID)
 		return false
 	}
@@ -656,7 +663,7 @@ func (t *TorrentManager) canCapacityHandle() bool {
 	for {
 		count, err := t.rd.GetActiveTorrentCount()
 		if err != nil {
-			t.log.Errorf("Cannot get active downloads count: %v", err)
+			t.log.Warnf("Cannot get active downloads count: %v", err)
 			if retryCount >= maxRetries {
 				t.log.Error("Max retries reached. Exiting.")
 				return false
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 4551039..55d3e3d 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -47,14 +47,14 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 
 	torrent, _ := t.TorrentMap.Get(accessKey)
 	if torrent == nil {
-		log.Errorf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
+		log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 
 	file, _ := torrent.SelectedFiles.Get(filename)
 	if file == nil {
-		log.Errorf("Cannot find file from path %s", requestPath)
+		log.Warnf("Cannot find file from path %s", requestPath)
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
@@ -66,7 +66,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 
 	if file.Link == "" {
 		// This is a dead file, serve an alternate file
-		log.Errorf("File %s is not yet available, zurg is repairing the torrent", filename)
+		log.Warnf("File %s is not yet available, zurg is repairing the torrent", filename)
 		streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, t, c, log)
 		return
 	}
@@ -75,18 +75,18 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	resp := t.UnrestrictUntilOk(link)
 	if resp == nil {
 		go t.Repair(torrent.AccessKey)
-		log.Errorf("File %s is no longer available, torrent is marked for repair", file.Path)
+		log.Warnf("File %s is no longer available, torrent is marked for repair", file.Path)
 		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
 		return
 	} else if resp.Filename != filename {
 		actualExt := filepath.Ext(resp.Filename)
 		expectedExt := filepath.Ext(filename)
 		if actualExt != expectedExt && resp.Streamable != 1 {
-			log.Errorf("File extension mismatch: %s and %s", filename, resp.Filename)
+			log.Warnf("File was changed and is not streamable: %s and %s", filename, resp.Filename)
 			streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, t, c, log)
 			return
 		} else {
-			log.Errorf("Filename mismatch: %s and %s", filename, resp.Filename)
+			log.Warnf("Filename mismatch: %s and %s", filename, resp.Filename)
 		}
 	}
 	cache.Add(requestPath, resp.Download)
@@ -108,11 +108,11 @@ func streamFileToResponse(torrent *torrent.Torrent, url string, w http.ResponseW
 	}
 
 	// Create a custom HTTP client
-	client := zurghttp.NewHTTPClient(c.GetToken(), 10)
+	client := zurghttp.NewHTTPClient(c.GetToken(), 10, c)
 
 	resp, err := client.Do(req)
 	if err != nil {
-		log.Errorf("Error downloading file %v ; torrent is marked for repair", err)
+		log.Warnf("Cannot download file %v ; torrent is marked for repair", err)
 		if torrent != nil {
 			go t.Repair(torrent.AccessKey)
 		}
@@ -122,7 +122,7 @@ func streamFileToResponse(torrent *torrent.Torrent, url string, w http.ResponseW
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		log.Errorf("Received a %s status code ; torrent is marked for repair", resp.Status)
+		log.Warnf("Received a %s status code ; torrent is marked for repair", resp.Status)
 		if torrent != nil {
 			go t.Repair(torrent.AccessKey)
 		}
diff --git a/internal/universal/head.go b/internal/universal/head.go
index a623fb8..346839f 100644
--- a/internal/universal/head.go
+++ b/internal/universal/head.go
@@ -46,20 +46,20 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren
 
 	torrent, _ := t.TorrentMap.Get(accessKey)
 	if torrent == nil {
-		log.Errorf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
+		log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 
 	file, _ := torrent.SelectedFiles.Get(filename)
 	if file == nil {
-		log.Errorf("Cannot find file from path %s", requestPath)
+		log.Warnf("Cannot find file from path %s", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
 	if file.Link == "" {
 		// This is a dead file, serve an alternate file
-		log.Errorf("File %s is no longer available", filename)
+		log.Warnf("File %s is no longer available", filename)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
diff --git a/pkg/dav/response.go b/pkg/dav/response.go
index 14a1ca3..21ee25a 100644
--- a/pkg/dav/response.go
+++ b/pkg/dav/response.go
@@ -2,7 +2,7 @@ package dav
 
 func Directory(path string) Response {
 	return Response{
-		Href: customPathEscape(path),
+		Href: "/" + customPathEscape(path),
 		Propstat: PropStat{
 			Prop: Prop{
 				ResourceType: ResourceType{Value: "<d:collection/>"},
@@ -14,7 +14,7 @@ func Directory(path string) Response {
 
 func File(path string, fileSize int64, added string, link string) Response {
 	return Response{
-		Href: customPathEscape(path),
+		Href: "/" + customPathEscape(path),
 		Propstat: PropStat{
 			Prop: Prop{
 				ContentLength: fileSize,
diff --git a/pkg/http/client.go b/pkg/http/client.go
index 9aae92f..b5d2a6a 100644
--- a/pkg/http/client.go
+++ b/pkg/http/client.go
@@ -1,19 +1,32 @@
 package http
 
 import (
+	"io"
 	"net/http"
+	"strings"
 	"time"
+
+	"github.com/debridmediamanager.com/zurg/internal/config"
+	"github.com/debridmediamanager.com/zurg/pkg/logutil"
+	"go.uber.org/zap"
 )
 
 type HTTPClient struct {
 	Client          *http.Client
 	MaxRetries      int
 	Backoff         func(attempt int) time.Duration
-	CheckRespStatus func(resp *http.Response, err error) bool
+	CheckRespStatus func(resp *http.Response, err error, log *zap.SugaredLogger) bool
 	BearerToken     string
+	log             *zap.SugaredLogger
+	config          config.ConfigInterface
 }
 
 func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
+	if r.config != nil && strings.Contains(req.Host, "download.real-debrid.") {
+		if host := r.config.GetRandomPreferredHost(); host != "" {
+			req.Host = r.config.GetRandomPreferredHost()
+		}
+	}
 	if r.BearerToken != "" {
 		req.Header.Set("Authorization", "Bearer "+r.BearerToken)
 	}
@@ -21,7 +34,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 	var err error
 	for attempt := 0; attempt < r.MaxRetries; attempt++ {
 		resp, err = r.Client.Do(req)
-		if !r.CheckRespStatus(resp, err) {
+		if !r.CheckRespStatus(resp, err, r.log) {
 			return resp, err
 		}
 		time.Sleep(r.Backoff(attempt))
@@ -29,7 +42,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 	return resp, err
 }
 
-func NewHTTPClient(token string, maxRetries int) *HTTPClient {
+func NewHTTPClient(token string, maxRetries int, c config.ConfigInterface) *HTTPClient {
 	return &HTTPClient{
 		BearerToken: token,
 		Client:      &http.Client{},
@@ -37,15 +50,21 @@ func NewHTTPClient(token string, maxRetries int) *HTTPClient {
 		Backoff: func(attempt int) time.Duration {
 			return time.Duration(attempt) * time.Second
 		},
-		CheckRespStatus: func(resp *http.Response, err error) bool {
+		CheckRespStatus: func(resp *http.Response, err error, log *zap.SugaredLogger) bool {
 			if err != nil {
 				return true
 			}
 			if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+				body, _ := io.ReadAll(resp.Body)
+				body2, _ := io.ReadAll(resp.Request.Body)
+				log.Errorf("Received a %s %s from %s", resp.Status, string(body), resp.Request.URL)
+				log.Errorf("request %s %s", string(body2), resp.Request.URL)
 				return true
 			}
 			// no need to retry because the status code is 2XX
 			return false
 		},
+		log:    logutil.NewLogger().Named("http"),
+		config: c,
 	}
 }
diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go
index a13fd78..7261b72 100644
--- a/pkg/realdebrid/api.go
+++ b/pkg/realdebrid/api.go
@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/debridmediamanager.com/zurg/internal/config"
 	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
 	"go.uber.org/zap"
 )
@@ -19,9 +20,9 @@ type RealDebrid struct {
 	client *zurghttp.HTTPClient
 }
 
-func NewRealDebrid(accessToken string, log *zap.SugaredLogger) *RealDebrid {
+func NewRealDebrid(accessToken string, config config.ConfigInterface, log *zap.SugaredLogger) *RealDebrid {
 	maxRetries := 10
-	client := zurghttp.NewHTTPClient(accessToken, maxRetries)
+	client := zurghttp.NewHTTPClient(accessToken, maxRetries, nil)
 	return &RealDebrid{
 		log:    log,
 		client: client,
@@ -52,11 +53,6 @@ func (rd *RealDebrid) UnrestrictCheck(link string) (*UnrestrictResponse, error)
 		return nil, err
 	}
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		rd.log.Errorf("Received a %s from unrestrict check endpoint", resp.Status)
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	var response UnrestrictResponse
 	err = json.Unmarshal(body, &response)
 	if err != nil {
@@ -101,11 +97,7 @@ func (rd *RealDebrid) GetTorrents(customLimit int) ([]Torrent, int, error) {
 		}
 		defer resp.Body.Close()
 
-		// if status code is not 2xx, return error
-		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-			rd.log.Errorf("Received a %s from get torrents endpoint", resp.Status)
-			return nil, 0, fmt.Errorf("HTTP error: %s", resp.Status)
-		}
+		// if status code is not 2xx, return erro
 
 		var torrents []Torrent
 		decoder := json.NewDecoder(resp.Body)
@@ -154,11 +146,6 @@ func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) {
 		return nil, err
 	}
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		rd.log.Errorf("Received a %s from get info endpoint", resp.Status)
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	var response TorrentInfo
 	err = json.Unmarshal(body, &response)
 	if err != nil {
@@ -191,11 +178,6 @@ func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error {
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		rd.log.Errorf("Received a %s from select files endpoint", resp.Status)
-		return fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	rd.log.Debugf("Selected files %s for torrent id=%s", len(strings.Split(files, ",")), id)
 	return nil
 }
@@ -218,11 +200,6 @@ func (rd *RealDebrid) DeleteTorrent(id string) error {
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		rd.log.Errorf("Received a %s from delete torrent endpoint", resp.Status)
-		return fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	rd.log.Debugf("Deleted torrent with id=%s", id)
 	return nil
 }
@@ -251,11 +228,6 @@ func (rd *RealDebrid) AddMagnetHash(magnet string) (*MagnetResponse, error) {
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		rd.log.Errorf("Received a %s from add magnet endpoint", resp.Status)
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	var response MagnetResponse
 	err = json.NewDecoder(resp.Body).Decode(&response)
 	if err != nil {
@@ -285,11 +257,6 @@ func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, erro
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		rd.log.Errorf("Received a %s from active torrents endpoint", resp.Status)
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	var response ActiveTorrentCountResponse
 	err = json.NewDecoder(resp.Body).Decode(&response)
 	if err != nil {
@@ -324,11 +291,6 @@ func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
 		return nil, err
 	}
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		rd.log.Errorf("Received a %s from unrestrict link endpoint", resp.Status)
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	var response UnrestrictResponse
 	err = json.Unmarshal(body, &response)
 	if err != nil {
@@ -340,6 +302,6 @@ func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
 		return nil, fmt.Errorf("can't fetch first byte")
 	}
 
-	rd.log.Debugf("Unrestricted link %s into %s", link, response.Download)
+	// rd.log.Debugf("Unrestricted link %s into %s", link, response.Download)
 	return &response, nil
 }

From 531624ee822c6210dbbcfe3511bc4af23f879eca Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Sat, 11 Nov 2023 03:30:10 +0100
Subject: [PATCH 17/19] repair fixes

---
 internal/torrent/manager.go  | 20 +++++++++++---------
 pkg/http/client.go           |  9 ++-------
 pkg/realdebrid/api.go        |  3 +--
 pkg/realdebrid/unrestrict.go |  9 +++++++--
 4 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index 75c01c7..f56b4e2 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -18,8 +18,8 @@ import (
 )
 
 type TorrentManager struct {
-	TorrentMap      *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
-	repairMap       *orderedmap.OrderedMap[string, bool]     // accessKey -> bool
+	TorrentMap      *orderedmap.OrderedMap[string, *Torrent]  // accessKey -> Torrent
+	repairMap       *orderedmap.OrderedMap[string, time.Time] // accessKey -> time last repaired
 	requiredVersion string
 	rd              *realdebrid.RealDebrid
 	checksum        string
@@ -35,7 +35,7 @@ type TorrentManager struct {
 func NewTorrentManager(config config.ConfigInterface, rd *realdebrid.RealDebrid) *TorrentManager {
 	t := &TorrentManager{
 		TorrentMap:      orderedmap.NewOrderedMap[string, *Torrent](),
-		repairMap:       orderedmap.NewOrderedMap[string, bool](),
+		repairMap:       orderedmap.NewOrderedMap[string, time.Time](),
 		requiredVersion: "10.11.2023",
 		rd:              rd,
 		config:          config,
@@ -514,7 +514,7 @@ func (t *TorrentManager) repairAll() {
 
 		// when getting info, we mark it for repair if it's missing some links
 		if torrent.ForRepair {
-			t.log.Infof("There were less links than was expected on %s; fixing...", torrent.AccessKey)
+			t.log.Infof("Found torrent for repair %s", torrent.AccessKey)
 			t.Repair(torrent.AccessKey)
 			break // only repair the first one for repair and then move on
 		}
@@ -522,10 +522,12 @@ func (t *TorrentManager) repairAll() {
 }
 
 func (t *TorrentManager) Repair(accessKey string) {
-	if _, exists := t.repairMap.Get(accessKey); exists {
-		return
+	if lastRepair, ok := t.repairMap.Get(accessKey); ok {
+		if time.Since(lastRepair) < time.Duration(24*time.Hour) { // magic number: 24 hrs
+			return
+		}
 	}
-	t.repairMap.Set(accessKey, true)
+	t.repairMap.Set(accessKey, time.Now())
 
 	if !t.config.EnableRepair() {
 		t.log.Warn("Repair is disabled; if you do not have other zurg instances running, you should enable repair")
@@ -564,7 +566,7 @@ func (t *TorrentManager) Repair(accessKey string) {
 
 	// first solution: add the same selection, maybe it can be fixed by reinsertion?
 	if t.reinsertTorrent(torrent, "") {
-		t.log.Infof("Redownloaded torrent %s to repair it", torrent.AccessKey)
+		t.log.Infof("Successfully downloaded torrent %s to repair it", torrent.AccessKey)
 		return
 	}
 	// if all the selected files are missing but there are other streamable files
@@ -633,7 +635,7 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string)
 	}
 
 	if info.Status == "magnet_error" || info.Status == "error" || info.Status == "virus" || info.Status == "dead" {
-		t.log.Warnf("Redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status)
+		t.log.Warnf("The redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status)
 		t.rd.DeleteTorrent(newTorrentID)
 		return false
 	}
diff --git a/pkg/http/client.go b/pkg/http/client.go
index b5d2a6a..7885a5a 100644
--- a/pkg/http/client.go
+++ b/pkg/http/client.go
@@ -1,7 +1,6 @@
 package http
 
 import (
-	"io"
 	"net/http"
 	"strings"
 	"time"
@@ -54,14 +53,10 @@ func NewHTTPClient(token string, maxRetries int, c config.ConfigInterface) *HTTP
 			if err != nil {
 				return true
 			}
-			if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-				body, _ := io.ReadAll(resp.Body)
-				body2, _ := io.ReadAll(resp.Request.Body)
-				log.Errorf("Received a %s %s from %s", resp.Status, string(body), resp.Request.URL)
-				log.Errorf("request %s %s", string(body2), resp.Request.URL)
+			if resp.StatusCode == 429 {
 				return true
 			}
-			// no need to retry because the status code is 2XX
+			// no need to retry
 			return false
 		},
 		log:    logutil.NewLogger().Named("http"),
diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go
index 7261b72..6c8831d 100644
--- a/pkg/realdebrid/api.go
+++ b/pkg/realdebrid/api.go
@@ -153,7 +153,6 @@ func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) {
 		return nil, err
 	}
 
-	rd.log.Debugf("Fetched info for torrent id=%s", response.ID)
 	return &response, nil
 }
 
@@ -178,7 +177,7 @@ func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error {
 	}
 	defer resp.Body.Close()
 
-	rd.log.Debugf("Selected files %s for torrent id=%s", len(strings.Split(files, ",")), id)
+	rd.log.Debugf("Selected files %d for torrent id=%s", len(strings.Split(files, ",")), id)
 	return nil
 }
 
diff --git a/pkg/realdebrid/unrestrict.go b/pkg/realdebrid/unrestrict.go
index 56f0d43..72fee99 100644
--- a/pkg/realdebrid/unrestrict.go
+++ b/pkg/realdebrid/unrestrict.go
@@ -8,10 +8,15 @@ import (
 )
 
 func (rd *RealDebrid) UnrestrictUntilOk(link string) *UnrestrictResponse {
-	unrestrictFn := func() (*UnrestrictResponse, error) {
+	if link == "" {
+		return nil
+	}
+	unrestrictFn := func(link string) (*UnrestrictResponse, error) {
 		return rd.UnrestrictLink(link)
 	}
-	return retryUntilOk(unrestrictFn)
+	return retryUntilOk(func() (*UnrestrictResponse, error) {
+		return unrestrictFn(link)
+	})
 }
 
 func retryUntilOk[T any](fn func() (T, error)) T {

From aa6ce3662c30589d289a5bec63dc4d39ff5330f7 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Sat, 11 Nov 2023 03:55:06 +0100
Subject: [PATCH 18/19] Add network test

---
 cmd/zurg/main.go          |   5 ++
 pkg/realdebrid/network.go | 106 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 111 insertions(+)
 create mode 100644 pkg/realdebrid/network.go

diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index f63d6e5..1305f9a 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -18,6 +18,11 @@ import (
 )
 
 func main() {
+	if len(os.Args) > 1 && os.Args[1] == "networktest" {
+		realdebrid.RunTest()
+		return
+	}
+
 	log := logutil.NewLogger().Named("zurg")
 
 	config, configErr := config.LoadZurgConfig("./config.yml")
diff --git a/pkg/realdebrid/network.go b/pkg/realdebrid/network.go
new file mode 100644
index 0000000..c33e62d
--- /dev/null
+++ b/pkg/realdebrid/network.go
@@ -0,0 +1,106 @@
+package realdebrid
+
+import (
+	"fmt"
+	"os/exec"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type IPInfo struct {
+	Address string
+	Hops    int
+	Latency time.Duration
+}
+
+func traceroute(ip string) (int, time.Duration, error) {
+	cmd := exec.Command("traceroute", "-n", "-q", "1", "-w", "1", ip)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	output := string(out)
+	lines := strings.Split(output, "\n")
+
+	hopCount := len(lines) - 1
+
+	var latency time.Duration
+	if hopCount > 0 {
+		lastLine := lines[hopCount-1]
+		fmt.Println(lastLine)
+		parts := strings.Fields(lastLine)
+		if len(parts) >= 3 {
+			latencyValue, parseErr := strconv.ParseFloat(parts[2], 64)
+			if parseErr == nil {
+				latency = time.Duration(latencyValue * float64(time.Millisecond))
+			}
+		}
+	}
+
+	return hopCount, latency, nil
+}
+
+func RunTest() {
+	fmt.Println("Running network test...")
+
+	ips := []string{"20.download.real-debrid.cloud", "20.download.real-debrid.com", "21.download.real-debrid.cloud", "21.download.real-debrid.com", "22.download.real-debrid.cloud", "22.download.real-debrid.com", "23.download.real-debrid.cloud", "23.download.real-debrid.com", "30.download.real-debrid.cloud", "30.download.real-debrid.com", "31.download.real-debrid.cloud", "31.download.real-debrid.com", "32.download.real-debrid.cloud", "32.download.real-debrid.com", "34.download.real-debrid.cloud", "34.download.real-debrid.com", "40.download.real-debrid.cloud", "40.download.real-debrid.com", "41.download.real-debrid.cloud", "41.download.real-debrid.com", "42.download.real-debrid.cloud", "42.download.real-debrid.com", "43.download.real-debrid.cloud", "43.download.real-debrid.com", "44.download.real-debrid.cloud", "44.download.real-debrid.com", "45.download.real-debrid.cloud", "45.download.real-debrid.com", "50.download.real-debrid.cloud", "50.download.real-debrid.com", "51.download.real-debrid.cloud", "51.download.real-debrid.com", "52.download.real-debrid.cloud", "52.download.real-debrid.com", "53.download.real-debrid.cloud", "53.download.real-debrid.com", "54.download.real-debrid.cloud", "54.download.real-debrid.com", "55.download.real-debrid.cloud", "55.download.real-debrid.com", "56.download.real-debrid.cloud", "56.download.real-debrid.com", "57.download.real-debrid.cloud", "57.download.real-debrid.com", "58.download.real-debrid.cloud", "58.download.real-debrid.com", "59.download.real-debrid.cloud", "59.download.real-debrid.com", "60.download.real-debrid.cloud", "60.download.real-debrid.com", "61.download.real-debrid.cloud", "61.download.real-debrid.com", "62.download.real-debrid.cloud", "62.download.real-debrid.com", "63.download.real-debrid.cloud", "63.download.real-debrid.com", "64.download.real-debrid.cloud", "64.download.real-debrid.com", "65.download.real-debrid.cloud", "65.download.real-debrid.com", "66.download.real-debrid.cloud", "66.download.real-debrid.com", "67.download.real-debrid.cloud", "67.download.real-debrid.com", "68.download.real-debrid.cloud", "68.download.real-debrid.com", "69.download.real-debrid.cloud", "69.download.real-debrid.com", "hkg1.download.real-debrid.com", "lax1.download.real-debrid.com", "lon1.download.real-debrid.com", "mum1.download.real-debrid.com", "rbx.download.real-debrid.com", "sgp1.download.real-debrid.com", "tlv1.download.real-debrid.com", "tyo1.download.real-debrid.com"}
+
+	var wg sync.WaitGroup
+	infoChan := make(chan IPInfo, len(ips))
+	semaphore := make(chan struct{}, 10)
+
+	for _, ip := range ips {
+		wg.Add(1)
+		semaphore <- struct{}{}
+		go func(ip string) {
+			defer wg.Done()
+			hops, latency, err := traceroute(ip)
+			if err != nil {
+				fmt.Println("Error performing traceroute:", err)
+			} else {
+				infoChan <- IPInfo{Address: ip, Hops: hops, Latency: latency}
+			}
+			<-semaphore
+		}(ip)
+	}
+
+	wg.Wait()
+	close(infoChan)
+
+	var ipInfos []IPInfo
+	for info := range infoChan {
+		ipInfos = append(ipInfos, info)
+	}
+
+	sort.Slice(ipInfos, func(i, j int) bool {
+		if ipInfos[i].Hops == ipInfos[j].Hops {
+			return ipInfos[i].Latency < ipInfos[j].Latency
+		}
+		return ipInfos[i].Hops < ipInfos[j].Hops
+	})
+	var lowestLatency time.Duration
+	if len(ipInfos) > 0 {
+		lowestLatency = ipInfos[0].Latency
+		for _, info := range ipInfos {
+			if info.Latency < lowestLatency {
+				lowestLatency = info.Latency
+			}
+		}
+	}
+
+	latencyThreshold := lowestLatency + lowestLatency/3
+
+	var okIPs []IPInfo
+	for _, info := range ipInfos {
+		if info.Latency <= latencyThreshold {
+			okIPs = append(okIPs, info)
+		}
+	}
+	for _, info := range okIPs {
+		fmt.Printf("Host: %s, Hops: %d, Latency: %v\n", info.Address, info.Hops, info.Latency)
+	}
+}

From abf4183af59ed094a38fa92f7c4dafde12fe0ce8 Mon Sep 17 00:00:00 2001
From: Ben Sarmiento <ben.sarmiento@forto.com>
Date: Sat, 11 Nov 2023 04:21:35 +0100
Subject: [PATCH 19/19] polishing

---
 internal/dav/listing.go  | 4 ++++
 internal/http/listing.go | 5 ++++-
 pkg/http/client.go       | 2 +-
 pkg/realdebrid/api.go    | 2 +-
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/internal/dav/listing.go b/internal/dav/listing.go
index 0b7bcf9..b9bb1ce 100644
--- a/internal/dav/listing.go
+++ b/internal/dav/listing.go
@@ -39,6 +39,10 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 	}
 
 	if err != nil {
+		if strings.Contains(err.Error(), "cannot find") {
+			http.Error(w, "Not Found", http.StatusNotFound)
+			return
+		}
 		log.Errorf("Error processing request: %v", err)
 		http.Error(w, "Server error", http.StatusInternalServerError)
 		return
diff --git a/internal/http/listing.go b/internal/http/listing.go
index af7e399..ddf0db7 100644
--- a/internal/http/listing.go
+++ b/internal/http/listing.go
@@ -34,8 +34,11 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
-
 	if err != nil {
+		if strings.Contains(err.Error(), "cannot find") {
+			http.Error(w, "Not Found", http.StatusNotFound)
+			return
+		}
 		log.Errorf("Error processing request: %v", err)
 		http.Error(w, "Server error", http.StatusInternalServerError)
 		return
diff --git a/pkg/http/client.go b/pkg/http/client.go
index 7885a5a..342fd91 100644
--- a/pkg/http/client.go
+++ b/pkg/http/client.go
@@ -59,7 +59,7 @@ func NewHTTPClient(token string, maxRetries int, c config.ConfigInterface) *HTTP
 			// no need to retry
 			return false
 		},
-		log:    logutil.NewLogger().Named("http"),
+		log:    logutil.NewLogger().Named("client"),
 		config: c,
 	}
 }
diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go
index 6c8831d..5367d7f 100644
--- a/pkg/realdebrid/api.go
+++ b/pkg/realdebrid/api.go
@@ -177,7 +177,7 @@ func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error {
 	}
 	defer resp.Body.Close()
 
-	rd.log.Debugf("Selected files %d for torrent id=%s", len(strings.Split(files, ",")), id)
+	rd.log.Debugf("Started the download for torrent id=%s", len(strings.Split(files, ",")), id)
 	return nil
 }