inject metadata on the fly when downloading, refactoring a bit

docs/USER_GUIDE.md

@@ -24,4 +24,8 @@ Scroll down to the end of the preferences page. There you can see a management t
 
 soundcloak tries to keep the URL schemes same to SoundCloud's, so you can just replace `soundcloud.com` with your instance URL. For short links: `https://on.soundcloud.com/boiKDP46fayYDoVK9` -> `<instance>/on/boiKDP46fayYDoVK9`
 
-To automatically redirect, you can use [LibRedirect](https://libredirect.github.io/) extension. Soundcloak is supported 
+To automatically redirect, you can use [LibRedirect](https://libredirect.github.io/) extension. Soundcloak is supported 
+
+# Extra notes
+
+If you find music that you like, make sure to download it! Stuff that's on there may be deleted or changed at any moment, without any warning or ability to experience it again, unless you download it for yourself. Download button is available if `Restream` is enabled in backend config. You can configure audio preset for downloading in preferences page. For easily and quickly downloading entire users or playlists, you can use my tool [scrip](https://git.maid.zone/laptop/scrip)

lib/preferences/init.go

@@ -117,7 +117,7 @@ func Load(r *fiber.App) {
 			return err
 		}
 
-		c.Set("Content-Type", "text/html")
+		c.Request().Header.SetContentType("text/html")
 		return templates.Base("preferences", templates.Preferences(p), nil).Render(context.Background(), c)
 	})
 

lib/proxy_images/init.go

@@ -63,7 +63,7 @@ func Load(r *fiber.App) {
 			return err
 		}
 
-		c.Set("Content-Type", "image/jpeg")
+		c.Request().Header.SetContentType("image/jpeg")
 		c.Set("Cache-Control", cfg.ImageCacheControl)
 		//return c.Send(resp.Body())
 		pr := misc.AcquireProxyReader()

lib/restream/init.go

@@ -2,11 +2,8 @@ package restream
 
 import (
 	"bytes"
-	"encoding/binary"
 	"image"
-	"io"
 	"strings"
-	"sync"
 
 	_ "image/jpeg"
 	_ "image/png"
@@ -19,191 +16,8 @@ import (
 	"github.com/gcottom/mp4meta"
 	"github.com/gcottom/oggmeta"
 	"github.com/gofiber/fiber/v3"
-	"github.com/valyala/fasthttp"
 )
 
-const defaultPartsCapacity = 24
-
-type reader struct {
-	parts    [][]byte
-	leftover []byte
-	index    int
-	duration *uint32
-
-	req    *fasthttp.Request
-	resp   *fasthttp.Response
-	client *fasthttp.HostClient
-}
-
-var readerpool = sync.Pool{
-	New: func() any {
-		return &reader{}
-	},
-}
-
-func acquireReader() *reader {
-	return readerpool.Get().(*reader)
-}
-
-func clone(buf []byte) []byte {
-	out := make([]byte, len(buf))
-	copy(out, buf)
-	return out
-}
-
-var mvhd = []byte("mvhd")
-
-func fixDuration(data []byte, duration *uint32) {
-	i := bytes.Index(data, mvhd)
-	if i != -1 {
-		i += 20
-
-		bt := make([]byte, 4)
-		binary.BigEndian.PutUint32(bt, *duration)
-		copy(data[i:], bt)
-		// timescale is already 1000 in the files
-	}
-}
-
-func (r *reader) Setup(url string, aac bool, duration *uint32) error {
-	if r.req == nil {
-		r.req = fasthttp.AcquireRequest()
-	}
-
-	if r.resp == nil {
-		r.resp = fasthttp.AcquireResponse()
-	}
-
-	r.req.SetRequestURI(url)
-	r.req.Header.SetUserAgent(cfg.UserAgent)
-	r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
-
-	if aac {
-		r.client = misc.HlsAacClient
-		r.duration = duration
-	} else {
-		r.client = misc.HlsClient
-	}
-
-	err := sc.DoWithRetry(r.client, r.req, r.resp)
-	if err != nil {
-		return err
-	}
-
-	data, err := r.resp.BodyUncompressed()
-	if err != nil {
-		data = r.resp.Body()
-	}
-
-	if r.parts == nil {
-		misc.Log("make() r.parts")
-		r.parts = make([][]byte, 0, defaultPartsCapacity)
-	} else {
-		misc.Log(cap(r.parts), len(r.parts))
-	}
-	if aac {
-		// clone needed to mitigate memory skill issues here
-		for _, s := range bytes.Split(data, []byte{'\n'}) {
-			if len(s) == 0 {
-				continue
-			}
-			if s[0] == '#' {
-				if bytes.HasPrefix(s, []byte(`#EXT-X-MAP:URI="`)) {
-					r.parts = append(r.parts, clone(s[16:len(s)-1]))
-				}
-
-				continue
-			}
-
-			r.parts = append(r.parts, clone(s))
-		}
-	} else {
-		for _, s := range bytes.Split(data, []byte{'\n'}) {
-			if len(s) == 0 || s[0] == '#' {
-				continue
-			}
-
-			r.parts = append(r.parts, s)
-		}
-	}
-
-	return nil
-}
-
-func (r *reader) Close() error {
-	misc.Log("closed :D")
-	r.req.Reset()
-	r.resp.Reset()
-
-	r.leftover = nil
-	r.index = 0
-	r.parts = r.parts[:0]
-
-	readerpool.Put(r)
-	return nil
-}
-
-// you could prob make this a bit faster by concurrency (make a bunch of workers => make them download the parts => temporarily add them to a map => fully assemble the result => make reader.Read() read out the result as the parts are coming in) but whatever, fine for now
-func (r *reader) Read(buf []byte) (n int, err error) {
-	misc.Log("we read")
-	if len(r.leftover) != 0 {
-		h := len(buf)
-		if h > len(r.leftover) {
-			h = len(r.leftover)
-		}
-
-		n = copy(buf, r.leftover[:h])
-
-		if n > len(r.leftover) {
-			r.leftover = r.leftover[:0]
-		} else {
-			r.leftover = r.leftover[n:]
-		}
-
-		if n < len(buf) && r.index == len(r.parts) {
-			err = io.EOF
-		}
-
-		return
-	}
-
-	if r.index == len(r.parts) {
-		err = io.EOF
-		return
-	}
-
-	r.req.SetRequestURIBytes(r.parts[r.index])
-
-	err = sc.DoWithRetry(r.client, r.req, r.resp)
-	if err != nil {
-		return
-	}
-
-	data, err := r.resp.BodyUncompressed()
-	if err != nil {
-		data = r.resp.Body()
-	}
-
-	if r.index == 0 && r.duration != nil {
-		fixDuration(data, r.duration) // I'm guessing that mvhd will always be in first part
-	}
-
-	if len(data) > len(buf) {
-		n = copy(buf, data[:len(buf)])
-	} else {
-		n = copy(buf, data)
-	}
-
-	r.leftover = data[n:]
-	r.index++
-
-	if n < len(buf) && r.index == len(r.parts) {
-		err = io.EOF
-	}
-
-	return
-}
-
 type collector struct {
 	data []byte
 }
@@ -250,9 +64,10 @@ func Load(r *fiber.App) {
 			return err
 		}
 
-		c.Set("Content-Type", tr.Format.MimeType)
+		c.Request().Header.SetContentType(tr.Format.MimeType)
 		c.Set("Cache-Control", cfg.RestreamCacheControl)
 
+		r := acquireReader()
 		if isDownload {
 			if t.Artwork != "" {
 				t.Artwork = strings.Replace(t.Artwork, "t500x500", "original", 1)
@@ -260,8 +75,8 @@ func Load(r *fiber.App) {
 
 			switch audio {
 			case cfg.AudioMP3:
-				r := acquireReader()
-				if err := r.Setup(u, false, nil); err != nil {
+				err := r.Setup(u, false, nil)
+				if err != nil {
 					return err
 				}
 
@@ -275,70 +90,37 @@ func Load(r *fiber.App) {
 				tag.SetTitle(t.Title)
 
 				if t.Artwork != "" {
-					data, mime, err := t.DownloadImage()
+					r.req.SetRequestURI(t.Artwork)
+					r.req.Header.Del("Accept-Encoding")
+
+					err := sc.DoWithRetry(misc.ImageClient, r.req, r.resp)
 					if err != nil {
 						return err
 					}
 
-					tag.AddAttachedPicture(id3v2.PictureFrame{MimeType: mime, Picture: data, PictureType: id3v2.PTFrontCover, Encoding: id3v2.EncodingUTF8})
+					tag.AddAttachedPicture(id3v2.PictureFrame{MimeType: cfg.B2s(r.req.Header.ContentType()), Picture: r.req.Body(), PictureType: id3v2.PTFrontCover, Encoding: id3v2.EncodingUTF8})
+					r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
 				}
 
 				var col collector
 				tag.WriteTo(&col)
 				r.leftover = col.data
 
-				// id3 is quite flexible and the files streamed by soundcloud don't have it so its easy to restream the stuff like this
 				return c.SendStream(r)
-
 			case cfg.AudioOpus:
-				req := fasthttp.AcquireRequest()
-				defer fasthttp.ReleaseRequest(req)
-
-				req.SetRequestURI(u)
-				req.Header.SetUserAgent(cfg.UserAgent)
-				req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
-
-				resp := fasthttp.AcquireResponse()
-				defer fasthttp.ReleaseResponse(resp)
-
-				err = sc.DoWithRetry(misc.HlsClient, req, resp)
+				err := r.Setup(u, false, nil)
 				if err != nil {
 					return err
 				}
 
-				data, err := resp.BodyUncompressed()
+				r.req.SetRequestURIBytes(r.parts[0])
+				err = sc.DoWithRetry(r.client, r.req, r.resp)
 				if err != nil {
-					data = resp.Body()
+					return err
 				}
 
-				parts := make([][]byte, 0, defaultPartsCapacity)
-				for _, s := range bytes.Split(data, []byte{'\n'}) {
-					if len(s) == 0 || s[0] == '#' {
-						continue
-					}
-
-					parts = append(parts, s)
-				}
-
-				result := []byte{}
-
-				for _, part := range parts {
-					req.SetRequestURIBytes(part)
-
-					err = sc.DoWithRetry(misc.HlsClient, req, resp)
-					if err != nil {
-						return err
-					}
-
-					data, err = resp.BodyUncompressed()
-					if err != nil {
-						data = resp.Body()
-					}
-
-					result = append(result, data...)
-				}
-
-				tag, err := oggmeta.ReadOGG(bytes.NewReader(result))
+				r.index++
+				tag, err := oggmeta.ReadOGG(bytes.NewReader(r.resp.Body()))
 				if err != nil {
 					return err
 				}
@@ -351,82 +133,43 @@ func Load(r *fiber.App) {
 				tag.SetTitle(t.Title)
 
 				if t.Artwork != "" {
-					req.SetRequestURI(t.Artwork)
-					req.Header.Del("Accept-Encoding")
+					r.req.SetRequestURI(t.Artwork)
+					r.req.Header.Del("Accept-Encoding")
 
-					err := sc.DoWithRetry(misc.ImageClient, req, resp)
+					err := sc.DoWithRetry(misc.ImageClient, r.req, r.resp)
 					if err != nil {
 						return err
 					}
 
-					defer resp.CloseBodyStream()
-					parsed, _, err := image.Decode(resp.BodyStream())
+					parsed, _, err := image.Decode(r.resp.BodyStream())
+					r.resp.CloseBodyStream()
 					if err != nil {
 						return err
 					}
 
 					tag.SetCoverArt(&parsed)
+					r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
 				}
 
-				return tag.Save(c)
+				var col collector
+				tag.Save(&col)
+				r.leftover = col.data
+
+				return c.SendStream(r)
 			case cfg.AudioAAC:
-				req := fasthttp.AcquireRequest()
-				defer fasthttp.ReleaseRequest(req)
-
-				req.SetRequestURI(u)
-				req.Header.SetUserAgent(cfg.UserAgent)
-				req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
-
-				resp := fasthttp.AcquireResponse()
-				defer fasthttp.ReleaseResponse(resp)
-
-				err = sc.DoWithRetry(misc.HlsAacClient, req, resp)
+				err := r.Setup(u, true, &t.Duration)
 				if err != nil {
 					return err
 				}
 
-				data, err := resp.BodyUncompressed()
+				r.req.SetRequestURIBytes(r.parts[0])
+				err = sc.DoWithRetry(r.client, r.req, r.resp)
 				if err != nil {
-					data = resp.Body()
+					return err
 				}
 
-				parts := make([][]byte, 0, defaultPartsCapacity)
-				// clone needed to mitigate memory skill issues here
-				for _, s := range bytes.Split(data, []byte{'\n'}) {
-					if len(s) == 0 {
-						continue
-					}
-					if s[0] == '#' {
-						if bytes.HasPrefix(s, []byte(`#EXT-X-MAP:URI="`)) {
-							parts = append(parts, clone(s[16:len(s)-1]))
-						}
-
-						continue
-					}
-
-					parts = append(parts, clone(s))
-				}
-
-				result := []byte{}
-				for _, part := range parts {
-					req.SetRequestURIBytes(part)
-
-					err = sc.DoWithRetry(misc.HlsAacClient, req, resp)
-					if err != nil {
-						return err
-					}
-
-					data, err = resp.BodyUncompressed()
-					if err != nil {
-						data = resp.Body()
-					}
-
-					result = append(result, data...)
-				}
-
-				fixDuration(result, &t.Duration)
-
-				tag, err := mp4meta.ReadMP4(bytes.NewReader(result))
+				r.index++
+				tag, err := mp4meta.ReadMP4(bytes.NewReader(r.resp.Body()))
 				if err != nil {
 					return err
 				}
@@ -439,28 +182,32 @@ func Load(r *fiber.App) {
 				tag.SetTitle(t.Title)
 
 				if t.Artwork != "" {
-					req.SetRequestURI(t.Artwork)
-					req.Header.Del("Accept-Encoding")
+					r.req.SetRequestURI(t.Artwork)
+					r.req.Header.Del("Accept-Encoding")
 
-					err := sc.DoWithRetry(misc.ImageClient, req, resp)
+					err := sc.DoWithRetry(misc.ImageClient, r.req, r.resp)
 					if err != nil {
 						return err
 					}
 
-					defer resp.CloseBodyStream()
-					parsed, _, err := image.Decode(resp.BodyStream())
+					parsed, _, err := image.Decode(r.resp.BodyStream())
+					r.resp.CloseBodyStream()
 					if err != nil {
 						return err
 					}
 
 					tag.SetCoverArt(&parsed)
+					r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
 				}
 
-				return tag.Save(c)
+				var col collector
+				tag.Save(&col)
+				r.leftover = col.data
+
+				return c.SendStream(r)
 			}
 		}
 
-		r := acquireReader()
 		if audio == cfg.AudioAAC {
 			err = r.Setup(u, true, &t.Duration)
 		} else {

lib/restream/reader.go

@@ -0,0 +1,195 @@
+package restream
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"sync"
+
+	"git.maid.zone/stuff/soundcloak/lib/cfg"
+	"git.maid.zone/stuff/soundcloak/lib/misc"
+	"git.maid.zone/stuff/soundcloak/lib/sc"
+	"github.com/valyala/fasthttp"
+)
+
+const defaultPartsCapacity = 24
+
+type reader struct {
+	parts    [][]byte
+	leftover []byte
+	index    int
+	duration *uint32
+
+	req    *fasthttp.Request
+	resp   *fasthttp.Response
+	client *fasthttp.HostClient
+}
+
+var readerpool = sync.Pool{
+	New: func() any {
+		return &reader{}
+	},
+}
+
+func acquireReader() *reader {
+	return readerpool.Get().(*reader)
+}
+
+func clone(buf []byte) []byte {
+	out := make([]byte, len(buf))
+	copy(out, buf)
+	return out
+}
+
+var mvhd = []byte("mvhd")
+
+func fixDuration(data []byte, duration *uint32) {
+	i := bytes.Index(data, mvhd)
+	if i != -1 {
+		i += 20
+
+		bt := make([]byte, 4)
+		binary.BigEndian.PutUint32(bt, *duration)
+		copy(data[i:], bt)
+		// timescale is already 1000 in the files
+	}
+}
+
+func (r *reader) Setup(url string, aac bool, duration *uint32) error {
+	if r.req == nil {
+		r.req = fasthttp.AcquireRequest()
+	}
+
+	if r.resp == nil {
+		r.resp = fasthttp.AcquireResponse()
+	}
+
+	r.req.SetRequestURI(url)
+	r.req.Header.SetUserAgent(cfg.UserAgent)
+	r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
+
+	if aac {
+		r.client = misc.HlsAacClient
+		r.duration = duration
+	} else {
+		r.client = misc.HlsClient
+	}
+
+	err := sc.DoWithRetry(r.client, r.req, r.resp)
+	if err != nil {
+		return err
+	}
+
+	data, err := r.resp.BodyUncompressed()
+	if err != nil {
+		data = r.resp.Body()
+	}
+
+	if r.parts == nil {
+		misc.Log("make() r.parts")
+		r.parts = make([][]byte, 0, defaultPartsCapacity)
+	} else {
+		misc.Log(cap(r.parts), len(r.parts))
+	}
+	if aac {
+		// clone needed to mitigate memory skill issues here
+		for _, s := range bytes.Split(data, []byte{'\n'}) {
+			if len(s) == 0 {
+				continue
+			}
+			if s[0] == '#' {
+				if bytes.HasPrefix(s, []byte(`#EXT-X-MAP:URI="`)) {
+					r.parts = append(r.parts, clone(s[16:len(s)-1]))
+				}
+
+				continue
+			}
+
+			r.parts = append(r.parts, clone(s))
+		}
+	} else {
+		for _, s := range bytes.Split(data, []byte{'\n'}) {
+			if len(s) == 0 || s[0] == '#' {
+				continue
+			}
+
+			r.parts = append(r.parts, s)
+		}
+	}
+
+	return nil
+}
+
+func (r *reader) Close() error {
+	misc.Log("closed :D")
+	r.req.Reset()
+	r.resp.Reset()
+
+	r.leftover = nil
+	r.index = 0
+	r.parts = r.parts[:0]
+
+	readerpool.Put(r)
+	return nil
+}
+
+// you could prob make this a bit faster by concurrency (make a bunch of workers => make them download the parts => temporarily add them to a map => fully assemble the result => make reader.Read() read out the result as the parts are coming in) but whatever, fine for now
+func (r *reader) Read(buf []byte) (n int, err error) {
+	misc.Log("we read")
+	if len(r.leftover) != 0 {
+		h := len(buf)
+		if h > len(r.leftover) {
+			h = len(r.leftover)
+		}
+
+		n = copy(buf, r.leftover[:h])
+
+		if n > len(r.leftover) {
+			r.leftover = r.leftover[:0]
+		} else {
+			r.leftover = r.leftover[n:]
+		}
+
+		if n < len(buf) && r.index == len(r.parts) {
+			err = io.EOF
+		}
+
+		return
+	}
+
+	if r.index == len(r.parts) {
+		err = io.EOF
+		return
+	}
+
+	r.req.SetRequestURIBytes(r.parts[r.index])
+
+	err = sc.DoWithRetry(r.client, r.req, r.resp)
+	if err != nil {
+		return
+	}
+
+	data, err := r.resp.BodyUncompressed()
+	if err != nil {
+		data = r.resp.Body()
+	}
+
+	if r.index == 0 && r.duration != nil {
+		fixDuration(data, r.duration) // I'm guessing that mvhd will always be in first part
+	}
+
+	if len(data) > len(buf) {
+		n = copy(buf, data[:len(buf)])
+	} else {
+		n = copy(buf, data)
+	}
+
+	r.leftover = data[n:]
+	r.index++
+
+	if n < len(buf) && r.index == len(r.parts) {
+		err = io.EOF
+	}
+
+	return
+}

lib/sc/track.go

@@ -448,30 +448,6 @@ func GetTrackByID(cid string, id string) (Track, error) {
 	return t, nil
 }
 
-func (t Track) DownloadImage() ([]byte, string, error) {
-	req := fasthttp.AcquireRequest()
-	defer fasthttp.ReleaseRequest(req)
-
-	req.SetRequestURI(t.Artwork)
-	req.Header.SetUserAgent(cfg.UserAgent)
-	//req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") images not big enough to be compressed
-
-	resp := fasthttp.AcquireResponse()
-	defer fasthttp.ReleaseResponse(resp)
-
-	err := DoWithRetry(misc.ImageClient, req, resp)
-	if err != nil {
-		return nil, "", err
-	}
-
-	data, err := resp.BodyUncompressed()
-	if err != nil {
-		data = resp.Body()
-	}
-
-	return data, string(resp.Header.Peek("Content-Type")), nil
-}
-
 func (t Track) Href() string {
 	return "/" + t.Author.Permalink + "/" + t.Permalink
 }