From 931957d23cb5c483f62515b8d697e99874f34e70 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Mon, 23 Oct 2023 01:21:08 +0200 Subject: [PATCH 1/8] enable head requests --- rclone.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rclone.conf b/rclone.conf index d8e8f95..91d8ec8 100644 --- a/rclone.conf +++ b/rclone.conf @@ -1,5 +1,5 @@ [zurg] type = http url = http://zurg:9999/http -no_head = true +no_head = false no_slash = true From 693f8dde8a8f0ba34ed8821c4eb3991b3dc338af Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Mon, 23 Oct 2023 13:20:34 +0200 Subject: [PATCH 2/8] Update config templates --- config.yml.example | 31 ++++++------------------------- docker-compose.yml | 24 +++--------------------- 2 files changed, 9 insertions(+), 46 deletions(-) diff --git a/config.yml.example b/config.yml.example index 55ac8cd..e91c597 100644 --- a/config.yml.example +++ b/config.yml.example @@ -1,7 +1,7 @@ # Zurg configuration version zurg: v1 -token: YOUR_TOKEN_HERE +token: YOUR_RD_API_TOKEN # https://real-debrid.com/apitoken port: 9999 concurrent_workers: 10 check_for_changes_every_secs: 15 @@ -9,21 +9,17 @@ info_cache_time_hours: 12 # List of directory definitions and their filtering rules directories: - # Configuration for TV shows shows: group: media # directories on different groups have duplicates of the same torrent filters: - regex: /season[\s\.]?\d/i # Capture torrent names with the term 'season' in any case - - regex: /Saison[\s\.]?\d/i # For non-English namings - - regex: /stage[\s\.]?\d/i + - regex: /saison[\s\.]?\d/i # For non-English namings + - regex: /stagione[\s\.]?\d/i # if there's french, there should be italian too - regex: /s\d\d/i # Capture common season notations like S01, S02, etc. + - regex: /\btv/i # anything that has TV in it is a TV show, right? - contains: complete - contains: seasons - - id: ATUWVRF53X5DA - - contains_strict: PM19 - - contains_strict: Detective Conan Remastered - - contains_strict: Goblin Slayer # Configuration for movies movies: @@ -31,22 +27,7 @@ directories: filters: - regex: /.*/ # you cannot leave a directory without filters because it will not have any torrents in it - # Configuration for Dolby Vision content - "hd movies": - group: another - filters: - - regex: /\b2160|\b4k|\buhd|\bdovi|\bdolby.?vision|\bdv|\bremux/i # Matches abbreviations of 'dolby vision' - - "low quality": - group: another + "ALL MY STUFFS": + group: all # notice the group now is "all", which means it will have all the torrents of shows+movies combined because this directory is alone in this group filters: - regex: /.*/ - - # Configuration for children's content - kids: - group: kids - filters: - - contains: xxx # Ensures adult content is excluded - - id: XFPQ5UCMUVAEG # Specific inclusion by torrent ID - - id: VDRPYNRPQHEXC - - id: YELNX3XR5XJQM diff --git a/docker-compose.yml b/docker-compose.yml index 43a519f..1539ef0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3.8' services: zurg: - image: debridmediamanager/zurg:latest + image: ghcr.io/debridmediamanager/zurg-testing:latest restart: unless-stopped ports: - 9999:9999 @@ -18,7 +18,7 @@ services: PUID: 1000 PGID: 1000 volumes: - - ./media:/data:rshared + - /mnt/zurg:/data:rshared - ./rclone.conf:/config/rclone/rclone.conf cap_add: - SYS_ADMIN @@ -26,25 +26,7 @@ services: - apparmor:unconfined devices: - /dev/fuse:/dev/fuse:rwm - command: "mount zurg: /data --allow-other --uid=1000 --gid=1000 --dir-cache-time 10s --read-only" - - rclonerd: - image: itstoggle/rclone_rd:latest - restart: unless-stopped - environment: - TZ: Europe/Berlin - PUID: 1000 - PGID: 1000 - volumes: - - ./media2:/data:rshared - - ./rclone.conf:/config/rclone/rclone.conf - command: "mount rd: /data --allow-other --uid=1000 --gid=1000 --dir-cache-time 10s --read-only" - devices: - - /dev/fuse:/dev/fuse:rwm - cap_add: - - SYS_ADMIN - security_opt: - - apparmor:unconfined + command: "mount zurg: /data --allow-non-empty --allow-other --uid=1000 --gid=1000 --dir-cache-time 10s --read-only" volumes: zurgdata: From 6298830ea65ace7b8f574d5a164bf990050b0f3d Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Mon, 23 Oct 2023 16:13:04 +0200 Subject: [PATCH 3/8] Prepare materials for auto heal functionality --- internal/dav/response.go | 1 + internal/torrent/manager.go | 191 ++++++++++++++++++---------------- pkg/realdebrid/api.go | 199 ++++++++++++++++++++++++++++++------ pkg/realdebrid/types.go | 10 ++ pkg/realdebrid/util.go | 34 ++++++ 5 files changed, 312 insertions(+), 123 deletions(-) diff --git a/internal/dav/response.go b/internal/dav/response.go index a87ec58..a50d86f 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -52,6 +52,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (* for _, torrent := range torrents { for _, file := range torrent.SelectedFiles { if file.Link == "" { + // TODO: Fix this file log.Println("File has no link, skipping", file.Path) continue } diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 221c224..69cfdc2 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -22,6 +22,102 @@ type TorrentManager struct { workerPool chan bool } +// NewTorrentManager creates a new torrent manager +// it will fetch all torrents and their info in the background +// and store them in-memory +func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager { + handler := &TorrentManager{ + config: config, + cache: cache, + workerPool: make(chan bool, config.GetNumOfWorkers()), + } + + // Initialize torrents for the first time + handler.torrents = handler.getAll() + + for _, torrent := range handler.torrents { + go func(id string) { + handler.workerPool <- true + handler.getInfo(id) + <-handler.workerPool + time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting + }(torrent.ID) + } + + // Start the periodic refresh + go handler.refreshTorrents() + + return handler +} + +// GetByDirectory returns all torrents that have a file in the specified directory +func (t *TorrentManager) GetByDirectory(directory string) []Torrent { + var torrents []Torrent + for i := range t.torrents { + for _, dir := range t.torrents[i].Directories { + if dir == directory { + torrents = append(torrents, t.torrents[i]) + } + } + } + return torrents +} + +// RefreshInfo refreshes the info for a torrent +func (t *TorrentManager) RefreshInfo(torrentID string) { + filePath := fmt.Sprintf("data/%s.bin", torrentID) + // Check the last modified time of the .bin file + fileInfo, err := os.Stat(filePath) + if err == nil { + modTime := fileInfo.ModTime() + // If the file was modified less than an hour ago, don't refresh + if time.Since(modTime) < time.Duration(t.config.GetCacheTimeHours())*time.Hour { + return + } + err = os.Remove(filePath) + if err != nil && !os.IsNotExist(err) { // File doesn't exist or other error + log.Printf("Cannot remove file: %v\n", err) + } + } else if !os.IsNotExist(err) { // Error other than file not existing + log.Printf("Error checking file info: %v\n", err) + return + } + info := t.getInfo(torrentID) + log.Println("Refreshed info for", info.Name) +} + +// MarkFileAsDeleted marks a file as deleted +func (t *TorrentManager) MarkFileAsDeleted(torrent *Torrent, file *File) { + log.Println("Marking file as deleted", file.Path) + file.Link = "" + t.writeToFile(torrent.ID, torrent) +} + +// GetInfo returns the info for a torrent +func (t *TorrentManager) GetInfo(torrentID string) *Torrent { + for i := range t.torrents { + if t.torrents[i].ID == torrentID { + return &t.torrents[i] + } + } + return t.getInfo(torrentID) +} + +// getChecksum returns the checksum based on the total count and the first torrent's ID +func (t *TorrentManager) getChecksum() string { + torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1) + if err != nil { + log.Printf("Cannot get torrents: %v\n", err) + return t.checksum + } + if len(torrents) == 0 { + log.Println("Huh, no torrents returned") + return t.checksum + } + return fmt.Sprintf("%d-%s", totalCount, torrents[0].ID) +} + +// refreshTorrents periodically refreshes the torrents func (t *TorrentManager) refreshTorrents() { log.Println("Starting periodic refresh") for { @@ -73,47 +169,7 @@ func (t *TorrentManager) refreshTorrents() { } } -// NewTorrentManager creates a new torrent manager -// it will fetch all torrents and their info in the background -// and store them in-memory -func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager { - handler := &TorrentManager{ - config: config, - cache: cache, - workerPool: make(chan bool, config.GetNumOfWorkers()), - } - - // Initialize torrents for the first time - handler.torrents = handler.getAll() - - for _, torrent := range handler.torrents { - go func(id string) { - handler.workerPool <- true - handler.getInfo(id) - <-handler.workerPool - time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting - }(torrent.ID) - } - - // Start the periodic refresh - go handler.refreshTorrents() - - return handler -} - -func (t *TorrentManager) getChecksum() string { - torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1) - if err != nil { - log.Printf("Cannot get torrents: %v\n", err) - return t.checksum - } - if len(torrents) == 0 { - log.Println("Huh, no torrents returned") - return t.checksum - } - return fmt.Sprintf("%d-%s", totalCount, torrents[0].ID) -} - +// getAll returns all torrents func (t *TorrentManager) getAll() []Torrent { log.Println("Getting all torrents") @@ -159,40 +215,7 @@ func (t *TorrentManager) getAll() []Torrent { return torrentsV2 } -func (t *TorrentManager) GetByDirectory(directory string) []Torrent { - var torrents []Torrent - for i := range t.torrents { - for _, dir := range t.torrents[i].Directories { - if dir == directory { - torrents = append(torrents, t.torrents[i]) - } - } - } - return torrents -} - -func (t *TorrentManager) RefreshInfo(torrentID string) { - filePath := fmt.Sprintf("data/%s.bin", torrentID) - // Check the last modified time of the .bin file - fileInfo, err := os.Stat(filePath) - if err == nil { - modTime := fileInfo.ModTime() - // If the file was modified less than an hour ago, don't refresh - if time.Since(modTime) < time.Duration(t.config.GetCacheTimeHours())*time.Hour { - return - } - err = os.Remove(filePath) - if err != nil && !os.IsNotExist(err) { // File doesn't exist or other error - log.Printf("Cannot remove file: %v\n", err) - } - } else if !os.IsNotExist(err) { // Error other than file not existing - log.Printf("Error checking file info: %v\n", err) - return - } - info := t.getInfo(torrentID) - log.Println("Refreshed info for", info.Name) -} - +// getInfo returns the info for a torrent func (t *TorrentManager) getInfo(torrentID string) *Torrent { torrentFromFile := t.readFromFile(torrentID) if torrentFromFile != nil { @@ -290,21 +313,7 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { return torrent } -func (t *TorrentManager) MarkFileAsDeleted(torrent *Torrent, file *File) { - log.Println("Marking file as deleted", file.Path) - file.Link = "" - t.writeToFile(torrent.ID, torrent) -} - -func (t *TorrentManager) GetInfo(torrentID string) *Torrent { - for i := range t.torrents { - if t.torrents[i].ID == torrentID { - return &t.torrents[i] - } - } - return t.getInfo(torrentID) -} - +// getByID returns a torrent by its ID func (t *TorrentManager) getByID(torrentID string) *Torrent { for i := range t.torrents { if t.torrents[i].ID == torrentID { @@ -314,6 +323,7 @@ func (t *TorrentManager) getByID(torrentID string) *Torrent { return nil } +// writeToFile writes a torrent to a file func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) { filePath := fmt.Sprintf("data/%s.bin", torrentID) file, err := os.Create(filePath) @@ -327,6 +337,7 @@ func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) { dataEncoder.Encode(torrent) } +// readFromFile reads a torrent from a file func (t *TorrentManager) readFromFile(torrentID string) *Torrent { filePath := fmt.Sprintf("data/%s.bin", torrentID) fileInfo, err := os.Stat(filePath) diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 4150346..06050d6 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -3,6 +3,7 @@ package realdebrid import ( "bytes" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -87,39 +88,6 @@ func UnrestrictLink(accessToken, link string) (*UnrestrictResponse, error) { return &response, nil } -func canFetchFirstByte(url string) bool { - // Create a new HTTP request - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return false - } - - // Set the Range header to request only the first byte - req.Header.Set("Range", "bytes=0-0") - - // Execute the request - resp, err := http.DefaultClient.Do(req) - if err != nil { - return false - } - defer resp.Body.Close() - - // If server supports partial content - if resp.StatusCode == http.StatusPartialContent { - buffer := make([]byte, 1) - _, err := resp.Body.Read(buffer) - return err == nil - } - if resp.StatusCode != http.StatusOK { - return false - } - // If server doesn't support partial content, try reading the first byte and immediately close - buffer := make([]byte, 1) - _, err = resp.Body.Read(buffer) - resp.Body.Close() // Close immediately after reading - return err == nil -} - // GetTorrents returns all torrents, paginated // if customLimit is 0, the default limit of 2500 is used func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) { @@ -216,3 +184,168 @@ func GetTorrentInfo(accessToken, id string) (*Torrent, error) { return &response, nil } + +// SelectTorrentFiles selects files of a torrent to start it. +func SelectTorrentFiles(accessToken string, id string, files string) error { + // Prepare request data + data := url.Values{} + data.Set("files", files) + + // Construct request URL + reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/selectFiles/%s", id) + req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode())) + if err != nil { + return err + } + + // Set request headers + req.Header.Set("Authorization", "Bearer "+accessToken) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Handle response status codes + switch resp.StatusCode { + case http.StatusOK, http.StatusNoContent: + return nil // Success + case http.StatusAccepted: + return errors.New("action already done") + case http.StatusBadRequest: + return errors.New("bad request") + case http.StatusUnauthorized: + return errors.New("bad token (expired or invalid)") + case http.StatusForbidden: + return errors.New("permission denied (account locked or not premium)") + case http.StatusNotFound: + return errors.New("wrong parameter (invalid file id(s)) or unknown resource (invalid id)") + default: + return fmt.Errorf("unexpected HTTP error: %s", resp.Status) + } +} + +// DeleteTorrent deletes a torrent from the torrents list. +func DeleteTorrent(accessToken string, id string) error { + // Construct request URL + reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/delete/%s", id) + req, err := http.NewRequest("DELETE", reqURL, nil) + if err != nil { + return err + } + + // Set request headers + req.Header.Set("Authorization", "Bearer "+accessToken) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Handle response status codes + switch resp.StatusCode { + case http.StatusNoContent: + return nil // Success + case http.StatusUnauthorized: + return errors.New("bad token (expired or invalid)") + case http.StatusForbidden: + return errors.New("permission denied (account locked)") + case http.StatusNotFound: + return errors.New("unknown resource") + default: + return fmt.Errorf("unexpected HTTP error: %s", resp.Status) + } +} + +// AddMagnet adds a magnet link to download. +func AddMagnet(accessToken, magnet, host string) (*MagnetResponse, error) { + // Prepare request data + data := url.Values{} + data.Set("magnet", magnet) + data.Set("host", host) + + // Construct request URL + reqURL := "https://api.real-debrid.com/rest/1.0/torrents/addMagnet" + req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode())) + if err != nil { + return nil, err + } + + // Set request headers + req.Header.Set("Authorization", "Bearer "+accessToken) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Handle response status codes + switch resp.StatusCode { + case http.StatusCreated: + var response MagnetResponse + err := json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, err + } + return &response, nil + case http.StatusBadRequest: + return nil, errors.New("bad request") + case http.StatusUnauthorized: + return nil, errors.New("bad token (expired or invalid)") + case http.StatusForbidden: + return nil, errors.New("permission denied (account locked or not premium)") + case http.StatusServiceUnavailable: + return nil, errors.New("service unavailable") + default: + return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status) + } +} + +// GetActiveTorrentCount gets the number of currently active torrents and the current maximum limit. +func GetActiveTorrentCount(accessToken string) (*ActiveTorrentCountResponse, error) { + // Construct request URL + reqURL := "https://api.real-debrid.com/rest/1.0/torrents/activeCount" + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, err + } + + // Set request headers + req.Header.Set("Authorization", "Bearer "+accessToken) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // Handle response status codes + switch resp.StatusCode { + case http.StatusOK: + var response ActiveTorrentCountResponse + err := json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return nil, err + } + return &response, nil + case http.StatusUnauthorized: + return nil, errors.New("bad token (expired or invalid)") + case http.StatusForbidden: + return nil, errors.New("permission denied (account locked)") + default: + return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status) + } +} diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go index d5bf443..105b88d 100644 --- a/pkg/realdebrid/types.go +++ b/pkg/realdebrid/types.go @@ -29,3 +29,13 @@ type File struct { Bytes int64 `json:"bytes"` Selected int `json:"selected"` } + +type MagnetResponse struct { + ID string `json:"id"` + URI string `json:"uri"` +} + +type ActiveTorrentCountResponse struct { + DownloadingCount int `json:"nb"` + MaxNumberOfTorrents int `json:"limit"` +} diff --git a/pkg/realdebrid/util.go b/pkg/realdebrid/util.go index aaf4a54..4022b41 100644 --- a/pkg/realdebrid/util.go +++ b/pkg/realdebrid/util.go @@ -2,6 +2,7 @@ package realdebrid import ( "math" + "net/http" "strings" "time" ) @@ -18,3 +19,36 @@ func RetryUntilOk[T any](fn func() (T, error)) T { time.Sleep(delay) } } + +func canFetchFirstByte(url string) bool { + // Create a new HTTP request + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return false + } + + // Set the Range header to request only the first byte + req.Header.Set("Range", "bytes=0-0") + + // Execute the request + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false + } + defer resp.Body.Close() + + // If server supports partial content + if resp.StatusCode == http.StatusPartialContent { + buffer := make([]byte, 1) + _, err := resp.Body.Read(buffer) + return err == nil + } + if resp.StatusCode != http.StatusOK { + return false + } + // If server doesn't support partial content, try reading the first byte and immediately close + buffer := make([]byte, 1) + _, err = resp.Body.Read(buffer) + resp.Body.Close() // Close immediately after reading + return err == nil +} From 21cbb16b88ae461cc908da56ca69f67168629e6c Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Mon, 23 Oct 2023 20:01:55 +0200 Subject: [PATCH 4/8] Implement autoheal feature --- internal/dav/getfile.go | 18 +-- internal/dav/response.go | 3 +- internal/http/get.go | 18 +-- internal/torrent/manager.go | 256 ++++++++++++++++++++++++++++-------- pkg/realdebrid/api.go | 8 +- pkg/realdebrid/types.go | 1 + 6 files changed, 218 insertions(+), 86 deletions(-) diff --git a/internal/dav/getfile.go b/internal/dav/getfile.go index 35fa408..37be857 100644 --- a/internal/dav/getfile.go +++ b/internal/dav/getfile.go @@ -64,23 +64,19 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent } resp := realdebrid.RetryUntilOk(unrestrictFn) if resp == nil { - // TODO: Readd the file - // when unrestricting fails, it means the file is not available anymore, but still in their database - // if it's the only file, tough luck - // if it's the only file, try to readd it - // delete the old one, add a new one log.Println("Cannot unrestrict link", link, filenameV2) t.MarkFileAsDeleted(torrent, file) http.Error(w, "Cannot find file", http.StatusNotFound) return } if resp.Filename != filenameV2 { - // TODO: Redo the logic to handle mismatch - // [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar - // Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv - // Action: schedule a "cleanup" job for the parent torrent - // If the file extension changed, that means it's a different file - log.Println("Filename mismatch", resp.Filename, filenameV2) + actualExt := filepath.Ext(resp.Filename) + expectedExt := filepath.Ext(filenameV2) + if actualExt != expectedExt { + log.Println("File extension mismatch", resp.Filename, filenameV2) + } else { + log.Println("Filename mismatch", resp.Filename, filenameV2) + } } cache.Add(requestPath, resp.Download) http.Redirect(w, r, resp.Download, http.StatusFound) diff --git a/internal/dav/response.go b/internal/dav/response.go index a50d86f..3c3e7d7 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -52,8 +52,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (* for _, torrent := range torrents { for _, file := range torrent.SelectedFiles { if file.Link == "" { - // TODO: Fix this file - log.Println("File has no link, skipping", file.Path) + log.Println("File has no link, skipping (repairing links take time)", file.Path) continue } diff --git a/internal/http/get.go b/internal/http/get.go index 39fa57d..8a40562 100644 --- a/internal/http/get.go +++ b/internal/http/get.go @@ -141,23 +141,19 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent } resp := realdebrid.RetryUntilOk(unrestrictFn) if resp == nil { - // TODO: Readd the file - // when unrestricting fails, it means the file is not available anymore, but still in their database - // if it's the only file, tough luck - // if it's the only file, try to readd it - // delete the old one, add a new one log.Println("Cannot unrestrict link", link, filenameV2) t.MarkFileAsDeleted(torrent, file) http.Error(w, "Cannot find file", http.StatusNotFound) return } if resp.Filename != filenameV2 { - // TODO: Redo the logic to handle mismatch - // [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar - // Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv - // Action: schedule a "cleanup" job for the parent torrent - // If the file extension changed, that means it's a different file - log.Println("Filename mismatch", resp.Filename, filenameV2) + actualExt := filepath.Ext(resp.Filename) + expectedExt := filepath.Ext(filenameV2) + if actualExt != expectedExt { + log.Println("File extension mismatch", resp.Filename, filenameV2) + } else { + log.Println("Filename mismatch", resp.Filename, filenameV2) + } } cache.Add(requestPath, resp.Download) http.Redirect(w, r, resp.Download, http.StatusFound) diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 69cfdc2..735af0a 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -4,6 +4,7 @@ import ( "encoding/gob" "fmt" "log" + "math" "os" "strings" "sync" @@ -91,6 +92,8 @@ func (t *TorrentManager) MarkFileAsDeleted(torrent *Torrent, file *File) { log.Println("Marking file as deleted", file.Path) file.Link = "" t.writeToFile(torrent.ID, torrent) + log.Println("Healing a single file in the torrent", torrent.Name) + t.heal(torrent.ID, []File{*file}) } // GetInfo returns the info for a torrent @@ -114,7 +117,7 @@ func (t *TorrentManager) getChecksum() string { log.Println("Huh, no torrents returned") return t.checksum } - return fmt.Sprintf("%d-%s", totalCount, torrents[0].ID) + return fmt.Sprintf("%d-%s-%v", totalCount, torrents[0].ID, torrents[0].Progress == 100) } // refreshTorrents periodically refreshes the torrents @@ -221,9 +224,11 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { if torrentFromFile != nil { torrent := t.getByID(torrentID) if torrent != nil { - torrent.SelectedFiles = torrentFromFile.SelectedFiles + if len(torrentFromFile.SelectedFiles) == len(torrent.Links) { + torrent.SelectedFiles = torrentFromFile.SelectedFiles + return torrent + } } - return torrent } log.Println("Getting info for", torrentID) info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrentID) @@ -242,62 +247,9 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { }) } if len(selectedFiles) != len(info.Links) { - // TODO: This means some files have expired - // we need to 'fix' this torrent then, at least the missing selected files log.Println("Some links has expired for", info.Name) - - type Result struct { - Response *realdebrid.UnrestrictResponse - } - - resultsChan := make(chan Result, len(info.Links)) - var wg sync.WaitGroup - - // Limit concurrency - sem := make(chan struct{}, t.config.GetNumOfWorkers()) - - for _, link := range info.Links { - wg.Add(1) - sem <- struct{}{} // Acquire semaphore - go func(lnk string) { - defer wg.Done() - defer func() { <-sem }() // Release semaphore - - unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { - return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk) - } - resp := realdebrid.RetryUntilOk(unrestrictFn) - if resp != nil { - resultsChan <- Result{Response: resp} - } - }(link) - } - - go func() { - wg.Wait() - close(sem) - close(resultsChan) - }() - - for result := range resultsChan { - found := false - for i := range selectedFiles { - if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) { - selectedFiles[i].Link = result.Response.Link - found = true - } - } - if !found { - selectedFiles = append(selectedFiles, File{ - File: realdebrid.File{ - Path: result.Response.Filename, - Bytes: result.Response.Filesize, - Selected: 1, - }, - Link: result.Response.Link, - }) - } - } + selectedFiles = t.organizeChaos(info, selectedFiles) + t.heal(torrentID, selectedFiles) } else { for i, link := range info.Links { selectedFiles[i].Link = link @@ -365,3 +317,191 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent { return &torrent } + +func (t *TorrentManager) reinsertTorrent(oldTorrentID string, missingFiles string, deleteIfFailed bool) bool { + torrent := t.GetInfo(oldTorrentID) + if torrent == nil { + return false + } + + if missingFiles == "" { + var selection string + for _, file := range torrent.SelectedFiles { + if file.Link == "" { + selection += fmt.Sprintf("%d,", file.ID) + } + } + if selection == "" { + return false + } + missingFiles = selection[:len(selection)-1] + } + + // reinsert torrent + resp, err := realdebrid.AddMagnetHash(t.config.GetToken(), torrent.Hash) + if err != nil { + log.Printf("Cannot reinsert torrent: %v\n", err) + return false + } + newTorrentID := resp.ID + err = realdebrid.SelectTorrentFiles(t.config.GetToken(), newTorrentID, missingFiles) + if err != nil { + log.Printf("Cannot select files on reinserted torrent: %v\n", err) + } + + if deleteIfFailed { + if err != nil { + realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID) + return false + } + time.Sleep(1 * time.Second) + // see if the torrent is ready + info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), newTorrentID) + if err != nil { + log.Printf("Cannot get info on reinserted torrent: %v\n", err) + if deleteIfFailed { + realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID) + } + return false + } + time.Sleep(1 * time.Second) + + if info.Progress != 100 { + log.Printf("Torrent is not cached anymore, %d%%\n", info.Progress) + realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID) + return false + } + if len(info.Links) != len(torrent.SelectedFiles) { + log.Printf("It doesn't fix the problem, got %d but we need %d\n", len(info.Links), len(torrent.SelectedFiles)) + realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID) + return false + } + log.Println("Reinsertion successful, deleting old torrent") + realdebrid.DeleteTorrent(t.config.GetToken(), oldTorrentID) + } + return true +} + +func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) []File { + type Result struct { + Response *realdebrid.UnrestrictResponse + } + + resultsChan := make(chan Result, len(info.Links)) + var wg sync.WaitGroup + + // Limit concurrency + sem := make(chan struct{}, t.config.GetNumOfWorkers()) + + for _, link := range info.Links { + wg.Add(1) + sem <- struct{}{} // Acquire semaphore + go func(lnk string) { + defer wg.Done() + defer func() { <-sem }() // Release semaphore + + unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { + return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk) + } + resp := realdebrid.RetryUntilOk(unrestrictFn) + if resp != nil { + resultsChan <- Result{Response: resp} + } + }(link) + } + + go func() { + wg.Wait() + close(sem) + close(resultsChan) + }() + + for result := range resultsChan { + found := false + for i := range selectedFiles { + if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) { + selectedFiles[i].Link = result.Response.Link + found = true + } + } + if !found { + selectedFiles = append(selectedFiles, File{ + File: realdebrid.File{ + Path: result.Response.Filename, + Bytes: result.Response.Filesize, + Selected: 1, + }, + Link: result.Response.Link, + }) + } + } + + return selectedFiles +} + +func (t *TorrentManager) heal(torrentID string, selectedFiles []File) { + // max waiting time is 45 minutes + const maxRetries = 50 + const baseDelay = 1 * time.Second + const maxDelay = 60 * time.Second + retryCount := 0 + for { + count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken()) + if err != nil { + log.Printf("Cannot get active torrent count: %v\n", err) + if retryCount >= maxRetries { + log.Println("Max retries reached. Exiting.") + return + } + delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay + if delay > maxDelay { + delay = maxDelay + } + time.Sleep(delay) + retryCount++ + continue + } + + if count.DownloadingCount < count.MaxNumberOfTorrents { + log.Printf("We can still add a new torrent, %d/%d\n", count.DownloadingCount, count.MaxNumberOfTorrents) + break + } + + if retryCount >= maxRetries { + log.Println("Max retries reached. Exiting.") + return + } + delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay + if delay > maxDelay { + delay = maxDelay + } + time.Sleep(delay) + retryCount++ + } + + // now we can get the missing files + half := len(selectedFiles) / 2 + missingFiles1 := getMissingFiles(0, half, selectedFiles) + missingFiles2 := getMissingFiles(half, len(selectedFiles), selectedFiles) + + // first solution: add the same selection, maybe it can be fixed by reinsertion? + success := t.reinsertTorrent(torrentID, "", true) + if !success { + // if not, last resort: add only the missing files and do it in 2 batches + t.reinsertTorrent(torrentID, missingFiles1, false) + t.reinsertTorrent(torrentID, missingFiles2, false) + } +} + +func getMissingFiles(start, end int, files []File) string { + var missingFiles string + for i := start; i < end; i++ { + if files[i].File.Selected == 1 && files[i].ID != 0 && files[i].Link == "" { + missingFiles += fmt.Sprintf("%d,", files[i].ID) + } + } + if len(missingFiles) > 0 { + missingFiles = missingFiles[:len(missingFiles)-1] + } + return missingFiles +} diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 06050d6..968dde2 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -104,6 +104,7 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) { params := url.Values{} params.Set("page", fmt.Sprintf("%d", page)) params.Set("limit", fmt.Sprintf("%d", limit)) + params.Set("filter", "active") reqURL := baseURL + "?" + params.Encode() @@ -264,12 +265,11 @@ func DeleteTorrent(accessToken string, id string) error { } } -// AddMagnet adds a magnet link to download. -func AddMagnet(accessToken, magnet, host string) (*MagnetResponse, error) { +// AddMagnetHash adds a magnet link to download. +func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) { // Prepare request data data := url.Values{} - data.Set("magnet", magnet) - data.Set("host", host) + data.Set("magnet", fmt.Sprintf("magnet:?xt=urn:btih:%s", magnet)) // Construct request URL reqURL := "https://api.real-debrid.com/rest/1.0/torrents/addMagnet" diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go index 105b88d..a68cb0a 100644 --- a/pkg/realdebrid/types.go +++ b/pkg/realdebrid/types.go @@ -25,6 +25,7 @@ type Torrent struct { } type File struct { + ID int `json:"id"` Path string `json:"path"` Bytes int64 `json:"bytes"` Selected int `json:"selected"` From 03db4b0c00089f14bc7a350be24e2ff7403cd3f3 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Tue, 24 Oct 2023 04:53:26 +0200 Subject: [PATCH 5/8] refactor --- internal/dav/getfile.go | 2 +- internal/dav/propfind.go | 2 +- internal/dav/util.go | 15 --------------- internal/http/get.go | 17 +++-------------- 4 files changed, 5 insertions(+), 31 deletions(-) diff --git a/internal/dav/getfile.go b/internal/dav/getfile.go index 37be857..b42c7c5 100644 --- a/internal/dav/getfile.go +++ b/internal/dav/getfile.go @@ -38,7 +38,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent torrentName := segments[len(segments)-2] filename := segments[len(segments)-1] - torrents := findAllTorrentsWithName(t, baseDirectory, torrentName) + torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName) if torrents == nil { log.Println("Cannot find torrent", torrentName) http.Error(w, "Cannot find file", http.StatusNotFound) diff --git a/internal/dav/propfind.go b/internal/dav/propfind.go index 76b7bf5..f982f3b 100644 --- a/internal/dav/propfind.go +++ b/internal/dav/propfind.go @@ -92,7 +92,7 @@ func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Requ directory := path.Dir(requestPath) torrentName := path.Base(requestPath) - sameNameTorrents := findAllTorrentsWithName(t, directory, torrentName) + sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName) if len(sameNameTorrents) == 0 { return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath) } diff --git a/internal/dav/util.go b/internal/dav/util.go index aa40a77..d1670f1 100644 --- a/internal/dav/util.go +++ b/internal/dav/util.go @@ -2,10 +2,7 @@ package dav import ( "log" - "strings" "time" - - "github.com/debridmediamanager.com/zurg/internal/torrent" ) // convertRFC3339toRFC1123 converts a date from RFC3339 to RFC1123 @@ -17,15 +14,3 @@ func convertRFC3339toRFC1123(input string) string { } return t.Format("Mon, 02 Jan 2006 15:04:05 GMT") } - -// findAllTorrentsWithName finds all torrents in a given directory with a given name -func findAllTorrentsWithName(t *torrent.TorrentManager, directory, torrentName string) []torrent.Torrent { - matchingTorrents := make([]torrent.Torrent, 0, 10) - torrents := t.GetByDirectory(directory) - for i := range torrents { - if torrents[i].Name == torrentName || strings.HasPrefix(torrents[i].Name, torrentName) { - matchingTorrents = append(matchingTorrents, torrents[i]) - } - } - return matchingTorrents -} diff --git a/internal/http/get.go b/internal/http/get.go index 8a40562..f86f1a4 100644 --- a/internal/http/get.go +++ b/internal/http/get.go @@ -44,7 +44,7 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren torrentName := segments[len(segments)-2] filename := segments[len(segments)-1] - torrents := findAllTorrentsWithName(t, baseDirectory, torrentName) + torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName) if torrents == nil { log.Println("Cannot find torrent", torrentName, segments) http.Error(w, "Cannot find file", http.StatusNotFound) @@ -115,7 +115,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent torrentName := segments[len(segments)-2] filename := segments[len(segments)-1] - torrents := findAllTorrentsWithName(t, baseDirectory, torrentName) + torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName) if torrents == nil { log.Println("Cannot find torrent", torrentName) http.Error(w, "Cannot find file", http.StatusNotFound) @@ -172,17 +172,6 @@ func getFile(torrents []torrent.Torrent, filename, fragment string) (*torrent.To return nil, nil } -func findAllTorrentsWithName(t *torrent.TorrentManager, directory, torrentName string) []torrent.Torrent { - matchingTorrents := make([]torrent.Torrent, 0, 10) - torrents := t.GetByDirectory(directory) - for i := range torrents { - if torrents[i].Name == torrentName || strings.HasPrefix(torrents[i].Name, torrentName) { - matchingTorrents = append(matchingTorrents, torrents[i]) - } - } - return matchingTorrents -} - func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) { requestPath := path.Clean(r.URL.Path) @@ -258,7 +247,7 @@ func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Requ directory := path.Base(fullDir) torrentName := path.Base(requestPath) - sameNameTorrents := findAllTorrentsWithName(t, directory, torrentName) + sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName) if len(sameNameTorrents) == 0 { return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath) } From c643eb17514932890194523ce9d5b08b295cbd4a Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Tue, 24 Oct 2023 04:54:01 +0200 Subject: [PATCH 6/8] Remove filter=active --- pkg/realdebrid/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 968dde2..3d765b2 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -104,7 +104,7 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) { params := url.Values{} params.Set("page", fmt.Sprintf("%d", page)) params.Set("limit", fmt.Sprintf("%d", limit)) - params.Set("filter", "active") + // params.Set("filter", "active") reqURL := baseURL + "?" + params.Encode() From 16668b90a1d01070df134bf287cf136406fec690 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Tue, 24 Oct 2023 04:54:24 +0200 Subject: [PATCH 7/8] Add ForRepair prop --- internal/torrent/types.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/torrent/types.go b/internal/torrent/types.go index 50b3946..de3844d 100644 --- a/internal/torrent/types.go +++ b/internal/torrent/types.go @@ -6,6 +6,7 @@ type Torrent struct { realdebrid.Torrent Directories []string SelectedFiles []File + ForRepair bool } type File struct { From 003ba1eb515c5d4aad6e9dbb7519661c002e9a8a Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Tue, 24 Oct 2023 14:56:03 +0200 Subject: [PATCH 8/8] Autoheal functionality --- README.md | 126 +++------ config.yml.example | 1 + internal/config/load.go | 1 + internal/config/types.go | 1 + internal/config/v1.go | 4 + internal/dav/response.go | 3 +- internal/http/response.go | 3 +- internal/torrent/manager.go | 499 +++++++++++++++++++++++------------- 8 files changed, 362 insertions(+), 276 deletions(-) diff --git a/README.md b/README.md index ccbdff0..f64d856 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,29 @@ -# zurg +# zurg-testing -## Building +A self-hosted Real-Debrid webdav server written from scratch, alternative to rclone_rd -```bash -docker build -t ghcr.io/debridmediamanager/zurg:latest . -``` +## How to run zurg in 5 steps -This builds zurg +1. Clone this repo `git clone https://github.com/debridmediamanager/zurg-testing.git` +2. Add your token in `config.yml` +3. `sudo mkdir -p /mnt/zurg` +4. Run `docker compose up -d` +5. `time ls -1R /mnt/zurg` You're done! + +The server is also exposed to your localhost via port 9999. You can point [Infuse](https://firecore.com/infuse) or any webdav clients to it. + +> Note: I have only tested this in Mac and Linux + +## Why zurg? Why not rclone_rd? Why not Real-Debrid's own webdav? + +- Better performance than anything out there; changes in your library appear instantly (assuming Plex picks it up fast enough) +- You should be able to access every file even if the torrent names are the same so if you have a lot of these, you might notice that zurg will have more files compared to others (e.g. 2 torrents named "Simpsons" but have different seasons, zurg merges all contents in that directory) +- You can configure a flexible directory structure in `config.yml`; you can select individual torrents that should appear on a directory by the ID you see in [DMM](https://debridmediamanager.com/) +- If you've ever experienced Plex scanner being stuck on a file and thereby freezing Plex completely, it should not happen anymore because zurg does a comprehensive check if a torrent is dead or not ## config.yml -You need a `config.yml` created before you use zurg +You need a `config.yml` created before you can use zurg ```yaml # Zurg configuration version @@ -18,27 +31,26 @@ zurg: v1 token: YOUR_TOKEN_HERE port: 9999 -concurrent_workers: 10 -check_for_changes_every_secs: 15 -info_cache_time_hours: 12 +concurrent_workers: 10 # the higher the number the faster zurg runs through your library but too high and you will get rate limited +check_for_changes_every_secs: 15 # zurg polls real-debrid for changes in your library +info_cache_time_hours: 12 # how long do we want to check if a torrent is still alive or dead? 12 to 24 hours is good enough + +# repair fixes broken links, but it doesn't mean it will appear on the same location (especially if there's only 1 episode missing) +enable_repair: false # BEWARE! THERE CAN ONLY BE 1 INSTANCE OF ZURG THAT SHOULD REPAIR YOUR TORRENTS # List of directory definitions and their filtering rules directories: - # Configuration for TV shows shows: group: media # directories on different groups have duplicates of the same torrent filters: - regex: /season[\s\.]?\d/i # Capture torrent names with the term 'season' in any case - - regex: /Saison[\s\.]?\d/i # For non-English namings - - regex: /stage[\s\.]?\d/i + - regex: /saison[\s\.]?\d/i # For non-English namings + - regex: /stagione[\s\.]?\d/i # if there's french, there should be italian too - regex: /s\d\d/i # Capture common season notations like S01, S02, etc. + - regex: /\btv/i # anything that has TV in it is a TV show, right? - contains: complete - contains: seasons - - id: ATUWVRF53X5DA - - contains_strict: PM19 - - contains_strict: Detective Conan Remastered - - contains_strict: Goblin Slayer # Configuration for movies movies: @@ -46,87 +58,17 @@ directories: filters: - regex: /.*/ # you cannot leave a directory without filters because it will not have any torrents in it - # Configuration for Dolby Vision content - "hd movies": - group: another - filters: - - regex: /\b2160|\b4k|\buhd|\bdovi|\bdolby.?vision|\bdv|\bremux/i # Matches abbreviations of 'dolby vision' - - "low quality": - group: another + "ALL MY STUFFS": + group: all # notice the group now is "all", which means it will have all the torrents of shows+movies combined because this directory is alone in this group filters: - regex: /.*/ - # Configuration for children's content - kids: + "Kids": group: kids filters: - - contains: xxx # Ensures adult content is excluded - - id: XFPQ5UCMUVAEG # Specific inclusion by torrent ID + - not_contains: xxx # Ensures adult content is excluded + - id: XFPQ5UCMUVAEG # Specific inclusion by torrent ID - id: VDRPYNRPQHEXC - id: YELNX3XR5XJQM ``` - - -## Running - -### Standalone webdav server - -```bash -docker run -v ./config.yml:/app/config.yml -v zurgdata:/app/data -p 9999:9999 ghcr.io/debridmediamanager/zurg:latest -``` - -- Runs zurg on port 9999 on your localhost -- Make sure you have config.yml on the current directory -- It creates a `zurgdata` volume for the data files - -### with rclone - -You will need to create a `media` directory to make the rclone mount work. - -```yaml -version: '3.8' - -services: - zurg: - image: ghcr.io/debridmediamanager/zurg:latest - restart: unless-stopped - ports: - - 9999 - volumes: - - ./config.yml:/app/config.yml - - zurgdata:/app/data - - rclone: - image: rclone/rclone:latest - restart: unless-stopped - environment: - TZ: Europe/Berlin - PUID: 1000 - PGID: 1000 - volumes: - - ./media:/data:rshared - - ./rclone.conf:/config/rclone/rclone.conf - cap_add: - - SYS_ADMIN - security_opt: - - apparmor:unconfined - devices: - - /dev/fuse:/dev/fuse:rwm - command: "mount zurg: /data --allow-non-empty --allow-other --uid 1000 --gid 1000 --dir-cache-time 1s --read-only" - -volumes: - zurgdata: -``` - -Together with this `docker-compose.yml` you will need this `rclone.conf` as well on the same directory. - -``` -[zurg] -type = http -url = http://zurg:9999/http -no_head = false -no_slash = true - -``` diff --git a/config.yml.example b/config.yml.example index e91c597..1648b81 100644 --- a/config.yml.example +++ b/config.yml.example @@ -6,6 +6,7 @@ port: 9999 concurrent_workers: 10 check_for_changes_every_secs: 15 info_cache_time_hours: 12 +enable_repair: true # BEWARE! THERE CAN ONLY BE 1 INSTANCE OF ZURG THAT SHOULD REPAIR YOUR TORRENTS # List of directory definitions and their filtering rules directories: diff --git a/internal/config/load.go b/internal/config/load.go index 1d5f94d..3813711 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -13,6 +13,7 @@ type ConfigInterface interface { GetNumOfWorkers() int GetRefreshEverySeconds() int GetCacheTimeHours() int + EnableRepair() bool GetPort() string GetDirectories() []string MeetsConditions(directory, fileID, fileName string) bool diff --git a/internal/config/types.go b/internal/config/types.go index d455b6f..0d5f52d 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -7,4 +7,5 @@ type ZurgConfig struct { NumOfWorkers int `yaml:"concurrent_workers"` RefreshEverySeconds int `yaml:"check_for_changes_every_secs"` CacheTimeHours int `yaml:"info_cache_time_hours"` + CanRepair bool `yaml:"enable_repair"` } diff --git a/internal/config/v1.go b/internal/config/v1.go index fca267f..a673fa8 100644 --- a/internal/config/v1.go +++ b/internal/config/v1.go @@ -40,6 +40,10 @@ func (z *ZurgConfigV1) GetCacheTimeHours() int { return z.CacheTimeHours } +func (z *ZurgConfigV1) EnableRepair() bool { + return z.CanRepair +} + func (z *ZurgConfigV1) GetDirectories() []string { rootDirectories := make([]string, len(z.Directories)) i := 0 diff --git a/internal/dav/response.go b/internal/dav/response.go index 3c3e7d7..8c0bec2 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -1,7 +1,6 @@ package dav import ( - "log" "path/filepath" "github.com/debridmediamanager.com/zurg/internal/torrent" @@ -52,7 +51,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (* for _, torrent := range torrents { for _, file := range torrent.SelectedFiles { if file.Link == "" { - log.Println("File has no link, skipping (repairing links take time)", file.Path) + // log.Println("File has no link, skipping (repairing links take time)", file.Path) continue } diff --git a/internal/http/response.go b/internal/http/response.go index 9cdfa94..c390fc4 100644 --- a/internal/http/response.go +++ b/internal/http/response.go @@ -2,7 +2,6 @@ package http import ( "fmt" - "log" "path/filepath" "github.com/debridmediamanager.com/zurg/internal/torrent" @@ -42,7 +41,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (s for _, torrent := range torrents { for _, file := range torrent.SelectedFiles { if file.Link == "" { - log.Println("File has no link, skipping", file.Path) + // log.Println("File has no link, skipping", file.Path) continue } diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 735af0a..c22b401 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -17,6 +17,7 @@ import ( type TorrentManager struct { torrents []Torrent + inProgress []string checksum string config config.ConfigInterface cache *expirable.LRU[string, string] @@ -27,28 +28,54 @@ type TorrentManager struct { // it will fetch all torrents and their info in the background // and store them in-memory func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager { - handler := &TorrentManager{ + t := &TorrentManager{ config: config, cache: cache, workerPool: make(chan bool, config.GetNumOfWorkers()), } // Initialize torrents for the first time - handler.torrents = handler.getAll() + t.torrents = t.getFreshListFromAPI() + t.checksum = t.getChecksum() + // log.Println("First checksum", t.checksum) + go t.mapToDirectories() - for _, torrent := range handler.torrents { - go func(id string) { - handler.workerPool <- true - handler.getInfo(id) - <-handler.workerPool - time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting - }(torrent.ID) + var wg sync.WaitGroup + + for i := range t.torrents { + wg.Add(1) + go func(idx int) { + defer wg.Done() + t.workerPool <- true + t.addMoreInfo(&t.torrents[idx]) + <-t.workerPool + }(i) + } + + if t.config.EnableRepair() { + go t.repairAll(&wg) } // Start the periodic refresh - go handler.refreshTorrents() + go t.startRefreshJob() - return handler + return t +} + +func (t *TorrentManager) repairAll(wg *sync.WaitGroup) { + wg.Wait() + for _, torrent := range t.torrents { + if torrent.ForRepair { + log.Println("Issues detected on", torrent.Name, "; fixing...") + t.repair(torrent.ID, torrent.SelectedFiles) + } + if len(torrent.Links) == 0 { + // If the torrent has no links + // and already processing repair + // delete it! + realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID) + } + } } // GetByDirectory returns all torrents that have a file in the specified directory @@ -64,126 +91,145 @@ func (t *TorrentManager) GetByDirectory(directory string) []Torrent { return torrents } -// RefreshInfo refreshes the info for a torrent -func (t *TorrentManager) RefreshInfo(torrentID string) { - filePath := fmt.Sprintf("data/%s.bin", torrentID) - // Check the last modified time of the .bin file - fileInfo, err := os.Stat(filePath) - if err == nil { - modTime := fileInfo.ModTime() - // If the file was modified less than an hour ago, don't refresh - if time.Since(modTime) < time.Duration(t.config.GetCacheTimeHours())*time.Hour { - return - } - err = os.Remove(filePath) - if err != nil && !os.IsNotExist(err) { // File doesn't exist or other error - log.Printf("Cannot remove file: %v\n", err) - } - } else if !os.IsNotExist(err) { // Error other than file not existing - log.Printf("Error checking file info: %v\n", err) - return - } - info := t.getInfo(torrentID) - log.Println("Refreshed info for", info.Name) -} - // MarkFileAsDeleted marks a file as deleted func (t *TorrentManager) MarkFileAsDeleted(torrent *Torrent, file *File) { log.Println("Marking file as deleted", file.Path) file.Link = "" - t.writeToFile(torrent.ID, torrent) + t.writeToFile(torrent) log.Println("Healing a single file in the torrent", torrent.Name) - t.heal(torrent.ID, []File{*file}) + t.repair(torrent.ID, []File{*file}) } -// GetInfo returns the info for a torrent -func (t *TorrentManager) GetInfo(torrentID string) *Torrent { - for i := range t.torrents { - if t.torrents[i].ID == torrentID { - return &t.torrents[i] +// FindAllTorrentsWithName finds all torrents in a given directory with a given name +func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string) []Torrent { + var matchingTorrents []Torrent + torrents := t.GetByDirectory(directory) + for i := range torrents { + if torrents[i].Name == torrentName || strings.HasPrefix(torrents[i].Name, torrentName) { + matchingTorrents = append(matchingTorrents, torrents[i]) } } - return t.getInfo(torrentID) + return matchingTorrents } -// getChecksum returns the checksum based on the total count and the first torrent's ID -func (t *TorrentManager) getChecksum() string { - torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1) - if err != nil { - log.Printf("Cannot get torrents: %v\n", err) - return t.checksum +// findAllDownloadedFilesFromHash finds all files that were with a given hash +func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File { + var files []File + for _, torrent := range t.torrents { + if torrent.Hash == hash { + for _, file := range torrent.SelectedFiles { + if file.Link != "" { + files = append(files, file) + } + } + } } + return files +} + +type torrentsResponse struct { + torrents []realdebrid.Torrent + totalCount int +} + +func (t *TorrentManager) getChecksum() string { + torrentsChan := make(chan torrentsResponse) + countChan := make(chan int) + errChan := make(chan error, 2) // accommodate errors from both goroutines + + // GetTorrents request + go func() { + torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1) + if err != nil { + errChan <- err + return + } + torrentsChan <- torrentsResponse{torrents: torrents, totalCount: totalCount} + }() + + // GetActiveTorrentCount request + go func() { + count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken()) + if err != nil { + errChan <- err + return + } + countChan <- count.DownloadingCount + }() + + var torrents []realdebrid.Torrent + var totalCount, count int + + for i := 0; i < 2; i++ { + select { + case torrentsResp := <-torrentsChan: + torrents = torrentsResp.torrents + totalCount = torrentsResp.totalCount + case count = <-countChan: + case err := <-errChan: + log.Printf("Error: %v\n", err) + return "" + } + } + if len(torrents) == 0 { log.Println("Huh, no torrents returned") - return t.checksum + return "" } - return fmt.Sprintf("%d-%s-%v", totalCount, torrents[0].ID, torrents[0].Progress == 100) + + checksum := fmt.Sprintf("%d%s%d", totalCount, torrents[0].ID, count) + return checksum } -// refreshTorrents periodically refreshes the torrents -func (t *TorrentManager) refreshTorrents() { +// startRefreshJob periodically refreshes the torrents +func (t *TorrentManager) startRefreshJob() { log.Println("Starting periodic refresh") for { <-time.After(time.Duration(t.config.GetRefreshEverySeconds()) * time.Second) + checksum := t.getChecksum() if checksum == t.checksum { continue } - t.checksum = checksum t.cache.Purge() - newTorrents := t.getAll() + newTorrents := t.getFreshListFromAPI() + var wg sync.WaitGroup - // Identify removed torrents - for i := 0; i < len(t.torrents); i++ { - found := false - for _, newTorrent := range newTorrents { - if t.torrents[i].ID == newTorrent.ID { - found = true - break - } - } - if !found { - // Remove this torrent from the slice - t.torrents = append(t.torrents[:i], t.torrents[i+1:]...) - i-- // Decrement index since we modified the slice - } + for i := range newTorrents { + wg.Add(1) + go func(idx int) { + defer wg.Done() + t.workerPool <- true + t.addMoreInfo(&newTorrents[idx]) + <-t.workerPool + }(i) } + wg.Wait() - // Identify and handle added torrents - for _, newTorrent := range newTorrents { - found := false - for _, torrent := range t.torrents { - if newTorrent.ID == torrent.ID { - found = true - break - } - } - if !found { - t.torrents = append(t.torrents, newTorrent) - go func(id string) { - t.workerPool <- true - t.getInfo(id) - <-t.workerPool - time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting - }(newTorrent.ID) - } + // apply side effects + t.torrents = newTorrents + t.checksum = t.getChecksum() + // log.Println("Checksum changed", t.checksum) + if t.config.EnableRepair() { + go t.repairAll(&wg) } + go t.mapToDirectories() } } -// getAll returns all torrents -func (t *TorrentManager) getAll() []Torrent { - log.Println("Getting all torrents") - - torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 0) +// getFreshListFromAPI returns all torrents +func (t *TorrentManager) getFreshListFromAPI() []Torrent { + torrents, _, err := realdebrid.GetTorrents(t.config.GetToken(), 0) if err != nil { log.Printf("Cannot get torrents: %v\n", err) return nil } - t.checksum = fmt.Sprintf("%d-%s", totalCount, torrents[0].ID) + // convert to own internal type without SelectedFiles yet + // populate inProgress var torrentsV2 []Torrent + t.inProgress = t.inProgress[:0] // reset for _, torrent := range torrents { torrent.Name = strings.TrimSuffix(torrent.Name, "/") torrentV2 := Torrent{ @@ -191,52 +237,48 @@ func (t *TorrentManager) getAll() []Torrent { SelectedFiles: nil, } torrentsV2 = append(torrentsV2, torrentV2) - } - log.Printf("Fetched %d torrents", len(torrentsV2)) - version := t.config.GetVersion() - if version == "v1" { - configV1 := t.config.(*config.ZurgConfigV1) - groupMap := configV1.GetGroupMap() - for group, directories := range groupMap { - log.Printf("Processing directory group: %s\n", group) - var directoryMap = make(map[string]int) - for i := range torrents { - for _, directory := range directories { - if configV1.MeetsConditions(directory, torrentsV2[i].ID, torrentsV2[i].Name) { - torrentsV2[i].Directories = append(torrentsV2[i].Directories, directory) - directoryMap[directory]++ - break - } - } - } - log.Printf("Finished processing directory group: %v\n", directoryMap) + if torrent.Progress != 100 { + t.inProgress = append(t.inProgress, torrent.Hash) } } - log.Println("Finished mapping to groups") + log.Printf("Fetched %d torrents", len(torrentsV2)) return torrentsV2 } -// getInfo returns the info for a torrent -func (t *TorrentManager) getInfo(torrentID string) *Torrent { - torrentFromFile := t.readFromFile(torrentID) +// addMoreInfo updates the selected files for a torrent +func (t *TorrentManager) addMoreInfo(torrent *Torrent) { + // file cache + torrentFromFile := t.readFromFile(torrent.ID) if torrentFromFile != nil { - torrent := t.getByID(torrentID) - if torrent != nil { - if len(torrentFromFile.SelectedFiles) == len(torrent.Links) { - torrent.SelectedFiles = torrentFromFile.SelectedFiles - return torrent - } + // see if api data and file data still match + // then it means data is still usable + if len(torrentFromFile.Links) == len(torrent.Links) { + torrent.ForRepair = torrentFromFile.ForRepair + torrent.SelectedFiles = torrentFromFile.SelectedFiles + return } } - log.Println("Getting info for", torrentID) - info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrentID) + // no file data yet as it is still downloading + if torrent.Progress != 100 { + return + } + + log.Println("Getting info for", torrent.ID) + info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrent.ID) if err != nil { log.Printf("Cannot get info: %v\n", err) - return nil + return } + + // SelectedFiles is a subset of Files with only the selected ones + // it also has a Link field, which can be empty + // if it is empty, it means the file is no longer available + // Files+Links together are the same as SelectedFiles var selectedFiles []File + // if some Links are empty, we need to repair it + forRepair := false for _, file := range info.Files { if file.Selected == 0 { continue @@ -246,23 +288,29 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { Link: "", }) } - if len(selectedFiles) != len(info.Links) { - log.Println("Some links has expired for", info.Name) - selectedFiles = t.organizeChaos(info, selectedFiles) - t.heal(torrentID, selectedFiles) + if len(selectedFiles) > len(info.Links) && info.Progress == 100 { + log.Printf("Some links has expired for %s, %s: %d selected but only %d links\n", info.ID, info.Name, len(selectedFiles), len(info.Links)) + // chaotic file means RD will not output the desired file selection + // e.g. even if we select just a single mkv, it will output a rar + var isChaotic bool + selectedFiles, isChaotic = t.organizeChaos(info, selectedFiles) + if isChaotic { + log.Println("This torrent is unfixable, ignoring", info.Name, info.ID) + } else { + log.Println("Marking for repair", info.Name) + forRepair = true + } } else { + // all links are still intact! good! for i, link := range info.Links { selectedFiles[i].Link = link } } - torrent := t.getByID(torrentID) - if torrent != nil { - torrent.SelectedFiles = selectedFiles - } - if len(torrent.SelectedFiles) > 0 { - t.writeToFile(torrentID, torrent) - } - return torrent + // update the torrent with more data! + torrent.SelectedFiles = selectedFiles + torrent.ForRepair = forRepair + // update file cache + t.writeToFile(torrent) } // getByID returns a torrent by its ID @@ -276,8 +324,8 @@ func (t *TorrentManager) getByID(torrentID string) *Torrent { } // writeToFile writes a torrent to a file -func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) { - filePath := fmt.Sprintf("data/%s.bin", torrentID) +func (t *TorrentManager) writeToFile(torrent *Torrent) { + filePath := fmt.Sprintf("data/%s.bin", torrent.ID) file, err := os.Create(filePath) if err != nil { log.Fatalf("Failed creating file: %s", err) @@ -314,27 +362,25 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent { log.Fatalf("Failed decoding file: %s", err) return nil } - return &torrent } -func (t *TorrentManager) reinsertTorrent(oldTorrentID string, missingFiles string, deleteIfFailed bool) bool { - torrent := t.GetInfo(oldTorrentID) - if torrent == nil { - return false - } - +func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool { + // if missingFiles is not provided, look for missing files if missingFiles == "" { + log.Println("Reinserting whole torrent", torrent.Name) var selection string for _, file := range torrent.SelectedFiles { - if file.Link == "" { - selection += fmt.Sprintf("%d,", file.ID) - } + selection += fmt.Sprintf("%d,", file.ID) } if selection == "" { return false } - missingFiles = selection[:len(selection)-1] + if len(selection) > 0 { + missingFiles = selection[:len(selection)-1] + } + } else { + log.Printf("Reinserting %d missing files for %s", len(strings.Split(missingFiles, ",")), torrent.Name) } // reinsert torrent @@ -371,18 +417,19 @@ func (t *TorrentManager) reinsertTorrent(oldTorrentID string, missingFiles strin realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID) return false } + if len(info.Links) != len(torrent.SelectedFiles) { - log.Printf("It doesn't fix the problem, got %d but we need %d\n", len(info.Links), len(torrent.SelectedFiles)) + log.Printf("It doesn't fix the problem, got %d links but we need %d\n", len(info.Links), len(torrent.SelectedFiles)) realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID) return false } log.Println("Reinsertion successful, deleting old torrent") - realdebrid.DeleteTorrent(t.config.GetToken(), oldTorrentID) + realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID) } return true } -func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) []File { +func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) ([]File, bool) { type Result struct { Response *realdebrid.UnrestrictResponse } @@ -395,10 +442,10 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [ for _, link := range info.Links { wg.Add(1) - sem <- struct{}{} // Acquire semaphore + sem <- struct{}{} go func(lnk string) { defer wg.Done() - defer func() { <-sem }() // Release semaphore + defer func() { <-sem }() unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk) @@ -416,6 +463,7 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [ close(resultsChan) }() + isChaotic := false for result := range resultsChan { found := false for i := range selectedFiles { @@ -425,6 +473,8 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [ } } if !found { + // "chaos" file, we don't know where it belongs + isChaotic = true selectedFiles = append(selectedFiles, File{ File: realdebrid.File{ Path: result.Response.Filename, @@ -436,10 +486,111 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [ } } - return selectedFiles + return selectedFiles, isChaotic } -func (t *TorrentManager) heal(torrentID string, selectedFiles []File) { +func (t *TorrentManager) repair(torrentID string, selectedFiles []File) { + torrent := t.getByID(torrentID) + if torrent == nil { + return + } + + // check if it is already "being" repaired + found := false + for _, hash := range t.inProgress { + if hash == torrent.Hash { + found = true + break + } + } + if found { + log.Println("Repair in progress, skipping", torrentID) + return + } + + // check if it is already repaired + foundFiles := t.findAllDownloadedFilesFromHash(torrent.Hash) + var missingFiles []File + for _, sFile := range selectedFiles { + if sFile.Link == "" { + found := false + for _, fFile := range foundFiles { + if sFile.Path == fFile.Path { + found = true + break + } + } + if !found { + missingFiles = append(missingFiles, sFile) + } + } + } + if len(missingFiles) == 0 { + log.Println(torrent.Name, "is already repaired") + return + } + + // then we repair it! + log.Println("Repairing torrent", torrentID) + // check if we can still add more downloads + proceed := t.canCapacityHandle() + if !proceed { + log.Println("Cannot add more torrents, exiting") + return + } + + // first solution: add the same selection, maybe it can be fixed by reinsertion? + success := t.reinsertTorrent(torrent, "", true) + if !success { + // if not, last resort: add only the missing files and do it in 2 batches + half := len(missingFiles) / 2 + missingFiles1 := getFileIDs(missingFiles[:half]) + missingFiles2 := getFileIDs(missingFiles[half:]) + if missingFiles1 != "" { + t.reinsertTorrent(torrent, missingFiles1, false) + } + if missingFiles2 != "" { + t.reinsertTorrent(torrent, missingFiles2, false) + } + log.Println("Waiting for downloads to finish") + } +} + +func (t *TorrentManager) mapToDirectories() { + // Map to directories + version := t.config.GetVersion() + if version == "v1" { + configV1 := t.config.(*config.ZurgConfigV1) + groupMap := configV1.GetGroupMap() + for group, directories := range groupMap { + log.Printf("Processing directory group: %s\n", group) + var directoryMap = make(map[string]int) + for i := range t.torrents { + for _, directory := range directories { + if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].Name) { + // append to t.torrents[i].Directories if not yet there + found := false + for _, dir := range t.torrents[i].Directories { + if dir == directory { + found = true + break + } + } + if !found { + t.torrents[i].Directories = append(t.torrents[i].Directories, directory) + } + directoryMap[directory]++ + break + } + } + } + log.Printf("Directory group: %v\n", directoryMap) + } + } + log.Println("Finished mapping to directories") +} + +func (t *TorrentManager) canCapacityHandle() bool { // max waiting time is 45 minutes const maxRetries = 50 const baseDelay = 1 * time.Second @@ -451,7 +602,7 @@ func (t *TorrentManager) heal(torrentID string, selectedFiles []File) { log.Printf("Cannot get active torrent count: %v\n", err) if retryCount >= maxRetries { log.Println("Max retries reached. Exiting.") - return + return false } delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay if delay > maxDelay { @@ -464,12 +615,12 @@ func (t *TorrentManager) heal(torrentID string, selectedFiles []File) { if count.DownloadingCount < count.MaxNumberOfTorrents { log.Printf("We can still add a new torrent, %d/%d\n", count.DownloadingCount, count.MaxNumberOfTorrents) - break + return true } if retryCount >= maxRetries { log.Println("Max retries reached. Exiting.") - return + return false } delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay if delay > maxDelay { @@ -478,30 +629,18 @@ func (t *TorrentManager) heal(torrentID string, selectedFiles []File) { time.Sleep(delay) retryCount++ } - - // now we can get the missing files - half := len(selectedFiles) / 2 - missingFiles1 := getMissingFiles(0, half, selectedFiles) - missingFiles2 := getMissingFiles(half, len(selectedFiles), selectedFiles) - - // first solution: add the same selection, maybe it can be fixed by reinsertion? - success := t.reinsertTorrent(torrentID, "", true) - if !success { - // if not, last resort: add only the missing files and do it in 2 batches - t.reinsertTorrent(torrentID, missingFiles1, false) - t.reinsertTorrent(torrentID, missingFiles2, false) - } } -func getMissingFiles(start, end int, files []File) string { - var missingFiles string - for i := start; i < end; i++ { - if files[i].File.Selected == 1 && files[i].ID != 0 && files[i].Link == "" { - missingFiles += fmt.Sprintf("%d,", files[i].ID) +func getFileIDs(files []File) string { + var fileIDs string + for _, file := range files { + // this won't include the id=0 files that were "chaos" + if file.File.Selected == 1 && file.ID != 0 && file.Link == "" { + fileIDs += fmt.Sprintf("%d,", file.ID) } } - if len(missingFiles) > 0 { - missingFiles = missingFiles[:len(missingFiles)-1] + if len(fileIDs) > 0 { + fileIDs = fileIDs[:len(fileIDs)-1] } - return missingFiles + return fileIDs }