basic song downloader backend

.gitignore

@@ -4,4 +4,5 @@
 *.dwarf
 .env
 /crystalgauntlet.db
-/config.toml
+/config.toml
+/data/

README.md

@@ -10,7 +10,7 @@ you may need to head into `lib/` to fix deps. i'm Very sorry
 
 ## setup
 
-copy `.env.example` to `.env` and fill it out, same for `config.toml.example` -> `config.toml`
+copy `.env.example` to `.env` and fill it out, same for `config.example.toml` -> `config.toml`
 
 run `cake db:migrate` (must have [cake](https://github.com/axvm/cake/))
 

config.example.toml

@@ -0,0 +1,101 @@
+[general]
+# if this path is encountered during path traversal,
+# it will be removed. this is useful for instances
+# where your absolute domain path is not long enough
+# to replace boomlings.com, because you can then point
+# it at a different, longer path to fill the gap
+#
+# example:
+#   boomlings.com/database/
+#   example.com/aaaaaaaaaa/
+#               ^^^^^^^^^^^
+#
+# leaving blank will disable this
+append_path = ""
+
+[formatting]
+# whether to format dates as relative or absolute
+# "relative" = relative, "absolute" = absolute
+# do note that absolute times can result in uglier
+# times due to colons being forbiddne in certain
+# spots
+date = "relative"
+
+[accounts]
+# allow new accounts to be created
+allow_registration = true
+
+[voting]
+# allow votes to influence a level's difficulty when it
+# hasn't been set yet. when set to false, all unrated
+# levels will be NA
+allow_votes = true
+# same as above, but for demon difficulties
+# this will let people vote and influence a demon'S
+# difficulty past its original demon rating
+allow_demon_votes = true
+# the minimum amount of votes before a level's difficulty
+# will go from NA to the average
+min_votes = 10
+# same as above, but for demon ratings
+min_demon_votes = 10
+
+[levels]
+# prevent users from deleting their own levels
+# if they are rated
+prevent_deletion_rated = true
+# prevent users from deleting their own levels
+# if they are featured
+prevent_deletion_featured = true
+
+[songs]
+# allow custom songs in general to be used,
+# whether it be non-newgrounds or newgrounds ones
+allow_custom_songs = true
+# allow non-newgrounds custom songs to be used
+# on the server
+allow_nong_songs = true
+# pushes all non-newgrounds songs above an arbitrary
+# id to prevent collisions with newgrounds ids, meaning
+# all song ids that work in vanilla GD will work
+# on the server
+preserve_newgrounds_ids = true
+
+[songs.sources]
+# allow ALL sources that yt-dlp supports for music
+# this is a BAD idea for many reasons
+allow_all_sources = false
+
+# lets you support much more sites but may result in much
+# slower download speeds and more bandwidth. requires ffmpeg
+allow_transcoding = true
+
+# location of your yt-dlp binary. get one here: https://github.com/yt-dlp/yt-dlp/releases/tag/2022.11.11
+# defaults to checking through path
+ytdlp_binary = "/usr/bin/yt-dlp"
+
+# location of your ffmpeg binary. get one here: https://ffmpeg.org/download.html
+# allows for transcoding
+# defaults to checking through path
+ffmpeg_binary = "/usr/bin/ffmpeg"
+
+# leads to more stable downloads at the cost of
+# using up much more storage to store every song
+#
+# required for allow_transcoding
+proxy_downloads = true
+
+# expressed in seconds, doesn't affect NG
+max_duration = 600
+
+# see: https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md
+# not every source is supported, and most video sites will fail w/o transcoding enabled
+[songs.sources.youtube]
+allow = true
+
+[songs.sources.soundcloud]
+allow = true
+
+[songs.sources.generic]
+# direct URLs and similar
+allow = true

config.toml.example

@@ -1,49 +0,0 @@
-[general]
-# if this path is encountered during path traversal,
-# it will be removed. this is useful for instances
-# where your absolute domain path is not long enough
-# to replace boomlings.com, because you can then point
-# it at a different, longer path to fill the gap
-#
-# example:
-#   boomlings.com/database/
-#   example.com/aaaaaaaaaa/
-#               ^^^^^^^^^^^
-#
-# leaving blank will disable this
-append_path = ""
-
-[formatting]
-# whether to format dates as relative or absolute
-# "relative" = relative, "absolute" = absolute
-# do note that absolute times can result in uglier
-# times due to colons being forbiddne in certain
-# spots
-date = "relative"
-
-[accounts]
-# allow new accounts to be created
-allow_registration = true
-
-[voting]
-# allow votes to influence a level's difficulty when it
-# hasn't been set yet. when set to false, all unrated
-# levels will be NA
-allow_votes = true
-# same as above, but for demon difficulties
-# this will let people vote and influence a demon'S
-# difficulty past its original demon rating
-allow_demon_votes = true
-# the minimum amount of votes before a level's difficulty
-# will go from NA to the average
-min_votes = 10
-# same as above, but for demon ratings
-min_demon_votes = 10
-
-[levels]
-# prevent users from deleting their own levels
-# if they are rated
-prevent_deletion_rated = true
-# prevent users from deleting their own levels
-# if they are featured
-prevent_deletion_featured = true

src/crystal-gauntlet.cr

@@ -72,6 +72,9 @@ module CrystalGauntlet
       server.bind_unix(listen_on.to_s.sub("unix://",""))
     end
 
+    # for debugging
+    #Songs.reupload("https://soundcloud.com/koraii/encroachingdark", 123456)
+
     puts "Listening on #{listen_on.to_s}"
     server.listen
   end

src/lib/songs.cr

@@ -1,14 +1,76 @@
+require "json"
+
 include CrystalGauntlet
 
 module CrystalGauntlet::Songs
   extend self
 
+  GD_AUDIO_FORMAT = "mp3"
+
   def is_custom_song(id)
     id >= 50
   end
 
   def is_reuploaded_song(id)
-    # todo: make configurable
     id >= 5000000
   end
+
+  class Song
+    def initialize(name : String, author : String, size : Int32, download_url : String | Nil, normalized_url : String)
+      @name = name
+      @author = author
+      @size = size
+      @download_url = download_url
+    end
+  end
+
+  def is_source_allowed(source : String) : Bool
+    config_get("songs.allow_all_sources").as?(Bool) || config_get("songs.sources.#{source}.allow").as?(Bool) || false
+  end
+
+  def reupload(url : String, id : Int32) : Song | Nil
+    puts url
+
+    output = IO::Memory.new
+    # todo: ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ LOOK OUT FOR SHELL INJECTION BULLSHIT!!!!!!!!!!!!!!!!!!
+    Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-J", url], output: output)
+    output.close
+
+    puts output.to_s
+
+    metadata = JSON.parse(output.to_s)
+
+    if !is_source_allowed(metadata["extractor"].as_s? || "unknown")
+      raise "source forbidden: #{metadata["extractor"]}"
+    end
+
+    max_duration = config_get("songs.sources.max_duration").as?(Int64) || 0
+
+    if max_duration > 0
+      if !metadata["duration"]
+        raise "failed to determine track duration"
+      elsif metadata["duration"].as_f >= max_duration
+        raise "track goes above max track duration (#{max_duration}s)"
+      end
+    end
+
+    if config_get("songs.sources.allow_transcoding")
+      if !config_get("songs.sources.proxy_downloads").as?(Bool)
+        raise "can't download a song with transcoding but without proxying allowed"
+      end
+
+      canonical_url = metadata["webpage_url"].as_s? || metadata["original_url"].as_s? || url
+
+      target_path = Path.new("data", "#{id}.mp3")
+
+      Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", canonical_url], output: STDOUT, error: STDOUT)
+
+      size = File.size(target_path)
+
+      # todo: don't point to localhost
+      Song.new(metadata["fulltitle"].as_s? || metadata["title"].as_s? || "Song", metadata["uploader"].as_s? || "", size.to_i32, "http://localhost:8080/#{id}.mp3", canonical_url)
+    else
+      # todo
+    end
+  end
 end