From a7357c7f0d0083cf4ba10aec0923b48512978955 Mon Sep 17 00:00:00 2001 From: "Jill \"oatmealine\" Monoids" Date: Mon, 2 Jan 2023 12:31:55 +0300 Subject: [PATCH] polish up songs, refactor http server initialization --- config.example.toml | 11 ++++--- db/migrations/6_songs.sql | 2 +- src/crystal-gauntlet.cr | 50 +++++++++++++++---------------- src/lib/songs.cr | 63 ++++++++++++++++++++++++++++----------- 4 files changed, 77 insertions(+), 49 deletions(-) diff --git a/config.example.toml b/config.example.toml index ceefdc5..7b485f4 100644 --- a/config.example.toml +++ b/config.example.toml @@ -52,9 +52,6 @@ prevent_deletion_featured = true # allow custom songs in general to be used, # whether it be non-newgrounds or newgrounds ones allow_custom_songs = true -# allow non-newgrounds custom songs to be used -# on the server -allow_nong_songs = true # pushes all non-newgrounds songs above an arbitrary # id to prevent collisions with newgrounds ids, meaning # all song ids that work in vanilla GD will work @@ -85,11 +82,17 @@ ffmpeg_binary = "/usr/bin/ffmpeg" # required for allow_transcoding proxy_downloads = true -# expressed in seconds, doesn't affect NG +# expressed in seconds max_duration = 600 +# expressed in bytes +max_filesize = 10000000 # = 10MB + # see: https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md # not every source is supported, and most video sites will fail w/o transcoding enabled +[songs.sources.newgrounds] +allow = true + [songs.sources.youtube] allow = true diff --git a/db/migrations/6_songs.sql b/db/migrations/6_songs.sql index 559d8ca..e2c6a66 100644 --- a/db/migrations/6_songs.sql +++ b/db/migrations/6_songs.sql @@ -10,7 +10,7 @@ CREATE TABLE songs ( -- so this is a seperate table that's filled in for any given song once -- it's needed CREATE TABLE song_data ( - id SERIAL PRIMARY KEY references songs(id), + id SERIAL PRIMARY KEY references songs(id), name TEXT NOT NULL, author_id INTEGER NOT NULL references song_authors(id), diff --git a/src/crystal-gauntlet.cr b/src/crystal-gauntlet.cr index 219d154..366d646 100644 --- a/src/crystal-gauntlet.cr +++ b/src/crystal-gauntlet.cr @@ -1,4 +1,5 @@ require "http/server" +require "http/server/handler" require "uri" require "sqlite3" require "migrate" @@ -41,38 +42,35 @@ module CrystalGauntlet @@endpoints end - def self.run() - server = HTTP::Server.new do |context| + class GDHandler + include HTTP::Handler + + def call(context) # expunge trailing slashes path = context.request.path.chomp("/") - # todo: rethink life choices - if path.ends_with?(".mp3") - # todo: BIG NONO - # todo: path traversal exploits SCARY - file = File.open("./data#{path}", "r") - context.response.content_type = "audio/mp3" - IO.copy(file, context.response) - file.close + path = path.sub(config_get("general.append_path").as(String | Nil) || "", "") + + body = context.request.body + + if CrystalGauntlet.endpoints.has_key?(path) && body + func = CrystalGauntlet.endpoints[path] + value = func.call(body.gets_to_end) + context.response.content_type = "text/plain" + context.response.print value else - path = path.sub(config_get("general.append_path").as(String | Nil) || "", "") - - body = context.request.body - - if !body - puts "no body :(" - elsif @@endpoints.has_key?(path) - func = @@endpoints[path] - value = func.call(body.gets_to_end) - context.response.content_type = "text/plain" - context.response.print value - puts "#{path} -> #{value}" - else - context.response.respond_with_status(404, "endpoint not found") - puts "#{path} -> 404" - end + call_next(context) end end + end + + def self.run() + server = HTTP::Server.new([ + HTTP::ErrorHandler.new, + HTTP::LogHandler.new, + CrystalGauntlet::GDHandler.new, + HTTP::StaticFileHandler.new("data/", directory_listing = true) + ]) listen_on = URI.parse(ENV["LISTEN_ON"]? || "http://localhost:8080").normalize diff --git a/src/lib/songs.cr b/src/lib/songs.cr index 43e66ed..2d19892 100644 --- a/src/lib/songs.cr +++ b/src/lib/songs.cr @@ -25,13 +25,14 @@ module CrystalGauntlet::Songs end class SongMetadata - def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil) + def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil, size : Int32 | Nil) @name = name @author = author @normalized_url = normalized_url @source = source @author_url = author_url @duration = duration + @size = size end def name @@ -52,6 +53,9 @@ module CrystalGauntlet::Songs def duration @duration end + def size + @size + end end def is_source_allowed(source : String) : Bool @@ -81,25 +85,41 @@ module CrystalGauntlet::Songs (metadata["uploader"]? && metadata["uploader"].as_s?) || "", canonical_url, metadata["extractor"].as_s, - (metadata.["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url, - duration ? duration.to_i : nil + (metadata["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url, + duration ? duration.to_i : nil, + metadata["filesize"]? && metadata["filesize"].as_i? ) end + def get_artist_id(artist_name : String, artist_url : String, source : String) : Int32 + if source == "unknown" + return UNKNOWN_SONG_AUTHOR + end + if artist_name == "" + return UNKNOWN_SONG_AUTHOR + end + + begin + DATABASE.query_one("select id from song_authors where name = ? and url = ? and source = ?", artist_name, artist_url, source, as: {Int32}) + rescue + next_id = (DATABASE.scalar("select max(id) from song_authors").as(Int64 | Nil) || 0) + 1 + DATABASE.exec("insert into song_authors (id, source, name, url) values (?, ?, ?, ?)", next_id, source, artist_name, artist_url) + next_id.to_i + end + end + # name, author id, author name, size, download url # returns nil if song should be disabled # throws if something failed def fetch_song(song_id : Int32, get_download = false) : Tuple(String, Int32, String, Int32 | Nil, String | Nil) | Nil puts "fetching #{song_id}" if !config_get("songs.allow_custom_songs").as?(Bool) - puts "custom songs not allowed" return nil end # todo: this is kinda spaghetti metadata = nil author_id = nil - size = nil fetch_url = nil song_exists = false @@ -125,9 +145,8 @@ module CrystalGauntlet::Songs song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_one("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ?", song_id, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?}) fetch_url = download_url - size = song_size author_id = song_author_id - metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration) + metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size) else begin metadata = fetch_song_metadata(url.not_nil!) @@ -139,6 +158,7 @@ module CrystalGauntlet::Songs else DATABASE.exec("insert into songs (id, url, disabled) values (?, ?, 1)", song_id, url) end + return nil else if song_exists && url != metadata.normalized_url DATABASE.exec("update songs set url = ? where id = ?", metadata.normalized_url, song_id) @@ -151,31 +171,29 @@ module CrystalGauntlet::Songs song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_all("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ?", song_id, metadata.normalized_url, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})[0] fetch_url = download_url - size = song_size author_id = song_author_id - metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration) + metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size) end end end puts metadata.inspect - # todo: insert into song_data - # do checks to make sure this is a valid song max_duration = config_get("songs.sources.max_duration").as?(Int64) # todo - if (fetch_url || !get_download) && metadata && size && author_id + if (fetch_url || !get_download) && metadata && author_id # we're done! woo if fetch_url && fetch_url.starts_with?("./") # todo fetch_url = "localhost:8080/#{fetch_url[2..]}" end - return {metadata.name, author_id, metadata.author, size, fetch_url} + return {metadata.name, author_id, metadata.author, metadata.size, fetch_url} end metadata = metadata.not_nil! + new_size = nil if get_download if config_get("songs.sources.allow_transcoding") @@ -189,20 +207,29 @@ module CrystalGauntlet::Songs Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", metadata.normalized_url], output: STDOUT, error: STDOUT) - size = File.size(target_path).to_i + new_size = File.size(target_path).to_i + + # todo: get duration fetch_url = "./#{song_id}.mp3" else # todo raise "fetching songs without transcoding and proxying downloads currently unimplemented" end - - # todo: update song_data with size, duration and url end if !author_id + author_id = get_artist_id(metadata.author, metadata.source, metadata.author_url) + end + + if config_get("songs.sources.proxy_downloads") + if DATABASE.scalar("select count(*) from song_data where id = ?", song_id).as(Int64) > 0 + DATABASE.exec("update song_data set name = ?, author_id = ?, size = ? where id = ?", metadata.name, author_id, new_size || metadata.size, song_id) + else + DATABASE.exec("insert into song_data (id, name, author_id, source, size, duration, proxy_url) values (?, ?, ?, ?, ?, ?, ?)", song_id, metadata.name, author_id, metadata.source, metadata.size, metadata.duration, fetch_url) + end + else # todo - author_id = 1 end if fetch_url && fetch_url.starts_with?("./") @@ -210,6 +237,6 @@ module CrystalGauntlet::Songs # todo also: deduplicate this with similar block above? fetch_url = "localhost:8080/#{fetch_url[2..]}" end - return {metadata.name, author_id, metadata.author, size, fetch_url} + return {metadata.name, author_id, metadata.author, new_size || metadata.size, fetch_url} end end