diff --git a/.vscode/settings.json b/.vscode/settings.json index 91884f0..52dd165 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,14 @@ "crystal-lang.completion": true, "crystal-lang.hover": true, "crystal-lang.implementations": true, - "crystal-lang.mainFile": "${workspaceRoot}/src/crystal-gauntlet.cr" + "crystal-lang.mainFile": "${workspaceRoot}/src/crystal-gauntlet.cr", + "sqltools.connections": [ + { + "previewLimit": 50, + "driver": "SQLite", + "name": "crystal-gauntlet", + "database": "${workspaceFolder:crystal-gauntlet}/crystalgauntlet.db" + } + ], + "sqltools.useNodeRuntime": true } \ No newline at end of file diff --git a/db/migrations/6_songs.sql b/db/migrations/6_songs.sql index a3cae4e..559d8ca 100644 --- a/db/migrations/6_songs.sql +++ b/db/migrations/6_songs.sql @@ -1,13 +1,42 @@ -- +migrate up CREATE TABLE songs ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL, - author_id INTEGER NOT NULL, - author_name TEXT NOT NULL, - size INTEGER NOT NULL, -- in bytes - download TEXT NOT NULL, - disabled INTEGER NOT NULL DEFAULT 0 + id SERIAL PRIMARY KEY, + url TEXT NOT NULL, + disabled INTEGER NOT NULL DEFAULT 0, + uploaded_by INTEGER references accounts(id) ); +-- song data is fetched on-demand rather than whenever songs are created, +-- so this is a seperate table that's filled in for any given song once +-- it's needed +CREATE TABLE song_data ( + id SERIAL PRIMARY KEY references songs(id), + + name TEXT NOT NULL, + author_id INTEGER NOT NULL references song_authors(id), + + source TEXT NOT NULL DEFAULT "unknown", + size INTEGER, -- in bytes + duration INTEGER, + + -- this may contain an absolute url OR a relative one (starts with ./) + -- depending on if its local or remote + -- null indicates it should be re-fetched every time its queried + proxy_url TEXT, + + last_updated TEXT NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'now')) +); + +CREATE TABLE song_authors ( + id SERIAL PRIMARY KEY, + source TEXT NOT NULL DEFAULT "unknown", + name TEXT NOT NULL, + url TEXT NOT NULL +); + +INSERT INTO song_authors (id, name, source, url) VALUES (1, "", "unknown", ""); + -- +migrate down -DROP TABLE songs; \ No newline at end of file +DROP TABLE songs; +DROP TABLE song_data; +DROP TABLE song_authors; \ No newline at end of file diff --git a/shard.yml b/shard.yml index 5b04a56..96db2f6 100644 --- a/shard.yml +++ b/shard.yml @@ -10,6 +10,8 @@ targets: main: src/crystal-gauntlet.cr dependencies: + db: + github: crystal-lang/crystal-db sqlite3: github: crystal-lang/crystal-sqlite3 migrate: @@ -20,8 +22,6 @@ dependencies: toml: github: crystal-community/toml.cr branch: master - crystagiri: - github: madeindjs/crystagiri crystal: 1.6.2 diff --git a/src/crystal-gauntlet.cr b/src/crystal-gauntlet.cr index 08d20e7..219d154 100644 --- a/src/crystal-gauntlet.cr +++ b/src/crystal-gauntlet.cr @@ -46,20 +46,31 @@ module CrystalGauntlet # expunge trailing slashes path = context.request.path.chomp("/") - path = path.sub(config_get("general.append_path").as(String | Nil) || "", "") - body = context.request.body - - if !body - puts "no body :(" - elsif @@endpoints.has_key?(path) - func = @@endpoints[path] - value = func.call(body.gets_to_end) - context.response.content_type = "text/plain" - context.response.print value - puts "#{path} -> #{value}" + # todo: rethink life choices + if path.ends_with?(".mp3") + # todo: BIG NONO + # todo: path traversal exploits SCARY + file = File.open("./data#{path}", "r") + context.response.content_type = "audio/mp3" + IO.copy(file, context.response) + file.close else - context.response.respond_with_status(404, "endpoint not found") - puts "#{path} -> 404" + path = path.sub(config_get("general.append_path").as(String | Nil) || "", "") + + body = context.request.body + + if !body + puts "no body :(" + elsif @@endpoints.has_key?(path) + func = @@endpoints[path] + value = func.call(body.gets_to_end) + context.response.content_type = "text/plain" + context.response.print value + puts "#{path} -> #{value}" + else + context.response.respond_with_status(404, "endpoint not found") + puts "#{path} -> 404" + end end end diff --git a/src/endpoints/levels/getLevels.cr b/src/endpoints/levels/getLevels.cr index 25e14bd..accf559 100644 --- a/src/endpoints/levels/getLevels.cr +++ b/src/endpoints/levels/getLevels.cr @@ -132,7 +132,8 @@ CrystalGauntlet.endpoints["/getGJLevels21.php"] = ->(body : String): String { # todo: search query where_str = "where (#{queryParams.join(") and (")})" - query_base = "from levels join users on levels.user_id = users.id left join songs on levels.song_id = songs.id left join map_pack_links on map_pack_links.level_id = levels.id #{where_str} order by #{order}" + # todo: switch join users to left join to avoid losing levels to the shadow realm after a user vanishes + query_base = "from levels join users on levels.user_id = users.id left join map_pack_links on map_pack_links.level_id = levels.id #{where_str} order by #{order}" puts query_base @@ -144,99 +145,71 @@ CrystalGauntlet.endpoints["/getGJLevels21.php"] = ->(body : String): String { hash_data = [] of Tuple(Int32, Int32, Bool) - DATABASE.query "select levels.id, levels.name, levels.user_id, levels.description, levels.original, levels.game_version, levels.requested_stars, levels.version, levels.song_id, levels.length, levels.objects, levels.coins, levels.has_ldm, levels.two_player, levels.downloads, levels.likes, levels.difficulty, levels.community_difficulty, levels.demon_difficulty, levels.stars, levels.featured, levels.epic, levels.rated_coins, users.username, users.udid, users.account_id, users.registered, songs.name, songs.author_id, songs.author_name, songs.size, songs.disabled, songs.download #{query_base} limit #{levels_per_page} offset #{page_offset}" do |rs| - rs.each do - id = rs.read(Int32) - name = rs.read(String) - user_id = rs.read(Int32) - description = rs.read(String) - original = rs.read(Int32 | Nil) - game_version = rs.read(Int32) - requested_stars = rs.read(Int32 | Nil) - version = rs.read(Int32) - song_id = rs.read(Int32) - length = rs.read(Int32) - objects = rs.read(Int32) - coins = rs.read(Int32) - has_ldm = rs.read(Bool) - two_player = rs.read(Bool) - downloads = rs.read(Int32) - likes = rs.read(Int32) - set_difficulty_int = rs.read(Int32 | Nil) - set_difficulty = set_difficulty_int && LevelDifficulty.new(set_difficulty_int) - community_difficulty_int = rs.read(Int32 | Nil) - community_difficulty = community_difficulty_int && LevelDifficulty.new(community_difficulty_int) - difficulty = set_difficulty || community_difficulty - demon_difficulty_int = rs.read(Int32 | Nil) - demon_difficulty = demon_difficulty_int && DemonDifficulty.new(demon_difficulty_int) - stars = rs.read(Int32 | Nil) - featured = rs.read(Bool) - epic = rs.read(Bool) - rated_coins = rs.read(Bool) + # fucking help + DATABASE.query_all("select levels.id, levels.name, levels.user_id, levels.description, levels.original, levels.game_version, levels.requested_stars, levels.version, levels.song_id, levels.length, levels.objects, levels.coins, levels.has_ldm, levels.two_player, levels.downloads, levels.likes, levels.difficulty, levels.community_difficulty, levels.demon_difficulty, levels.stars, levels.featured, levels.epic, levels.rated_coins, users.username, users.udid, users.account_id, users.registered #{query_base} limit #{levels_per_page} offset #{page_offset}", as: {Int32, String, Int32, String, Int32 | Nil, Int32, Int32 | Nil, Int32, Int32, Int32, Int32, Int32, Bool, Bool, Int32, Int32, Int32 | Nil, Int32 | Nil, Int32 | Nil, Int32 | Nil, Bool, Bool, Bool, String, String | Nil, Int32 | Nil, Bool}).map() do |id, name, user_id, description, original, game_version, requested_stars, version, song_id, length, objects, coins, has_ldm, two_player, downloads, likes, set_difficulty_int, community_difficulty_int, demon_difficulty_int, stars, featured, epic, rated_coins, user_username, user_udid, user_account_id, user_registered| + set_difficulty = set_difficulty_int && LevelDifficulty.new(set_difficulty_int) + community_difficulty = community_difficulty_int && LevelDifficulty.new(community_difficulty_int) + difficulty = set_difficulty || community_difficulty + demon_difficulty = demon_difficulty_int && DemonDifficulty.new(demon_difficulty_int) - user_username = rs.read(String) - user_udid = rs.read(String | Nil) - user_account_id = rs.read(Int32 | Nil) - user_registered = rs.read(Bool) + # https://github.com/Cvolton/GMDprivateServer/blob/master/incl/levels/getGJLevels.php#L266 + results << Format.fmt_hash({ + 1 => id, + 2 => name, + 5 => version, + 6 => user_id, + 8 => 10, + 9 => difficulty ? difficulty.to_star_difficulty : 0, # 0=N/A 10=EASY 20=NORMAL 30=HARD 40=HARDER 50=INSANE 50=AUTO 50=DEMON + 10 => downloads, + 12 => !Songs.is_custom_song(song_id) ? song_id : 0, + 13 => game_version, + 14 => likes, + 17 => difficulty && difficulty.demon?, + # 0 for n/a, 10 for easy, 20, for medium, ... + 43 => (demon_difficulty || DemonDifficulty::Hard).to_demon_difficulty, + 25 => difficulty && difficulty.auto?, + 18 => stars || 0, + 19 => featured, + 42 => epic, + 45 => objects, + 3 => GDBase64.encode(description), + 15 => length, + 30 => original || 0, + 31 => two_player, + 37 => coins, + 38 => rated_coins, + 39 => requested_stars || 0, + 46 => 1, + 47 => 2, + 40 => has_ldm, + 35 => Songs.is_custom_song(song_id) ? song_id : 0, + }) - song_name = rs.read(String | Nil) - song_author_id = rs.read(Int32 | Nil) - song_author_name = rs.read(String | Nil) - song_size = rs.read(Int32 | Nil) - song_disabled = rs.read(Int32 | Nil) - song_download = rs.read(String | Nil) + users << "#{user_id}:#{user_username}:#{user_registered ? user_account_id : user_udid}" - # https://github.com/Cvolton/GMDprivateServer/blob/master/incl/levels/getGJLevels.php#L266 - results << Format.fmt_hash({ - 1 => id, - 2 => name, - 5 => version, - 6 => user_id, - 8 => 10, - 9 => difficulty ? difficulty.to_star_difficulty : 0, # 0=N/A 10=EASY 20=NORMAL 30=HARD 40=HARDER 50=INSANE 50=AUTO 50=DEMON - 10 => downloads, - 12 => !Songs.is_custom_song(song_id) ? song_id : 0, - 13 => game_version, - 14 => likes, - 17 => difficulty && difficulty.demon?, - # 0 for n/a, 10 for easy, 20, for medium, ... - 43 => (demon_difficulty || DemonDifficulty::Hard).to_demon_difficulty, - 25 => difficulty && difficulty.auto?, - 18 => stars || 0, - 19 => featured, - 42 => epic, - 45 => objects, - 3 => GDBase64.encode(description), - 15 => length, - 30 => original || 0, - 31 => two_player, - 37 => coins, - 38 => rated_coins, - 39 => requested_stars || 0, - 46 => 1, - 47 => 2, - 40 => has_ldm, - 35 => Songs.is_custom_song(song_id) ? song_id : 0, - }) - - users << "#{user_id}:#{user_username}:#{user_registered ? user_account_id : user_udid}" - - if Songs.is_custom_song(song_id) && song_disabled == 0 - songs << Format.fmt_song({ - 1 => song_id, - 2 => song_name, - 3 => song_author_id, - 4 => song_author_name, - 5 => song_size.not_nil! / (1000 * 1000), - 6 => "", - 10 => song_download, - 7 => "", - 8 => "1" - }) + if Songs.is_custom_song(song_id) + begin + song = Songs.fetch_song(song_id, false) + rescue + else + if song != nil + song_name, song_author_id, song_author_name, song_size, song_download = song.not_nil! + songs << Format.fmt_song({ + 1 => song_id, + 2 => song_name, + 3 => song_author_id, + 4 => song_author_name, + 5 => (song_size || 0) / (1000 * 1000), + 6 => "", + 10 => song_download || "", + 7 => "", + 8 => "1" + }) + end end - - hash_data << {id, stars || 0, rated_coins} end + + hash_data << {id, stars || 0, rated_coins} end # `${amount}:${offset}:${levelsPerPage}` diff --git a/src/endpoints/songs/getSongInfo.cr b/src/endpoints/songs/getSongInfo.cr index 8555b1b..2d36ccb 100644 --- a/src/endpoints/songs/getSongInfo.cr +++ b/src/endpoints/songs/getSongInfo.cr @@ -1,5 +1,4 @@ require "uri" -require "crystagiri" require "http/client" require "digest/sha256" @@ -17,71 +16,27 @@ CrystalGauntlet.endpoints["/getGJSongInfo.php"] = ->(body : String): String { song_id = params["songID"].to_i32 - DATABASE.query("select name, author_id, author_name, size, download, disabled from songs where id = ?", song_id) do |rs| - if rs.move_next - song_name = rs.read(String) - author_id = rs.read(Int32) - author_name = rs.read(String) - size = rs.read(Int32) - download = rs.read(String) - disabled = rs.read(Int32) - - if disabled == 1 - return "-2" - end + song = Songs.fetch_song(song_id, true) + if song != nil + begin + song_name, song_author_id, song_author_name, song_size, song_download = song.not_nil! + rescue + return "-1" + else return Format.fmt_song({ 1 => song_id, 2 => song_name, - 3 => author_id, - 4 => author_name, - 5 => size / (1000 * 1000), + 3 => song_author_id, + 4 => song_author_name, + 5 => (song_size || 0) / (1000 * 1000), 6 => "", - 10 => download, + 10 => song_download || "", 7 => "", - 8 => "0" + 8 => "1" }) end - end - - if Songs.is_reuploaded_song(song_id) - # todo - "-1" else - # todo: maybe use yt-dlp? for other sources too - doc = Crystagiri::HTML.from_url "https://www.newgrounds.com/audio/listen/#{song_id}" - - song_name = (doc.css("title") { |d| })[0].content - song_artist = (doc.css(".item-details-main > h4 > a") { |d| d })[0].content - song_url_str = (doc.css("script") { |d| }) - .map { |d| d.node.to_s.match(NEWGROUNDS_AUDIO_URL_REGEX) } - .reduce { |acc, d| acc || d } - .not_nil![1] - - # todo: proxy locally - song_url = unescape_string(song_url_str).split("?")[0].sub("https://", "http://") - - # todo: consider hashes? - size = 0 - - HTTP::Client.head(song_url) do |response| - size = response.headers["content-length"].to_i - end - - author_id = 9 # todo: what is this needed for? - - DATABASE.exec("insert into songs (id, name, author_id, author_name, size, download) values (?, ?, ?, ?, ?, ?)", song_id, song_name, author_id, song_artist, size, song_url) - - return Format.fmt_song({ - 1 => song_id, - 2 => song_name, - 3 => author_id, - 4 => song_artist, - 5 => size / (1000 * 1000), - 6 => "", - 10 => song_url, - 7 => "", - 8 => "0" - }) + return "-2" end } diff --git a/src/lib/songs.cr b/src/lib/songs.cr index f639fa1..43e66ed 100644 --- a/src/lib/songs.cr +++ b/src/lib/songs.cr @@ -1,4 +1,5 @@ require "json" +require "db" include CrystalGauntlet @@ -7,20 +8,49 @@ module CrystalGauntlet::Songs GD_AUDIO_FORMAT = "mp3" + # todo: make this configurable + REUPLOADED_SONG_ADD_ID = 5000000 + + CUSTOM_SONG_START = 50 + + # set in 6_songs.sql + UNKNOWN_SONG_AUTHOR = 1 + def is_custom_song(id) - id >= 50 + id >= CUSTOM_SONG_START end def is_reuploaded_song(id) - id >= 5000000 + id >= REUPLOADED_SONG_ADD_ID end - class Song - def initialize(name : String, author : String, size : Int32, download_url : String | Nil, normalized_url : String) + class SongMetadata + def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil) @name = name @author = author - @size = size - @download_url = download_url + @normalized_url = normalized_url + @source = source + @author_url = author_url + @duration = duration + end + + def name + @name + end + def author + @author + end + def normalized_url + @normalized_url + end + def source + @source + end + def author_url + @author_url + end + def duration + @duration end end @@ -28,49 +58,158 @@ module CrystalGauntlet::Songs config_get("songs.allow_all_sources").as?(Bool) || config_get("songs.sources.#{source}.allow").as?(Bool) || false end - def reupload(url : String, id : Int32) : Song | Nil - puts url + def get_file_path(song_id : Int32) + Path.new("data", "#{song_id}.mp3") + end + + # will raise errors + def fetch_song_metadata(url : String) : SongMetadata + puts "getting metadata for #{url}" output = IO::Memory.new # todo: ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ LOOK OUT FOR SHELL INJECTION BULLSHIT!!!!!!!!!!!!!!!!!! Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-J", url], output: output) output.close - puts output.to_s - metadata = JSON.parse(output.to_s) - if !is_source_allowed(metadata["extractor"].as_s? || "unknown") - raise "source forbidden: #{metadata["extractor"]}" + canonical_url = metadata["webpage_url"].as_s? || metadata["original_url"].as_s? || url + duration = metadata["duration"]? && (metadata["duration"].as_f? || metadata["duration"].as_i?) + + return SongMetadata.new( + (metadata["fulltitle"]? && metadata["fulltitle"].as_s?) || (metadata["title"]? && metadata["title"].as_s?) || url, + (metadata["uploader"]? && metadata["uploader"].as_s?) || "", + canonical_url, + metadata["extractor"].as_s, + (metadata.["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url, + duration ? duration.to_i : nil + ) + end + + # name, author id, author name, size, download url + # returns nil if song should be disabled + # throws if something failed + def fetch_song(song_id : Int32, get_download = false) : Tuple(String, Int32, String, Int32 | Nil, String | Nil) | Nil + puts "fetching #{song_id}" + if !config_get("songs.allow_custom_songs").as?(Bool) + puts "custom songs not allowed" + return nil end - max_duration = config_get("songs.sources.max_duration").as?(Int64) || 0 + # todo: this is kinda spaghetti + metadata = nil + author_id = nil + size = nil + fetch_url = nil - if max_duration > 0 - if !metadata["duration"] - raise "failed to determine track duration" - elsif metadata["duration"].as_f >= max_duration - raise "track goes above max track duration (#{max_duration}s)" + song_exists = false + url = nil + + begin + url, disabled = DATABASE.query_one("select url, disabled from songs where id = ?", song_id, as: {String, Bool}) + + if disabled + return nil + end + + song_exists = true + rescue + if config_get("songs.preserve_newgrounds_ids").as?(Bool) + url = "https://www.newgrounds.com/audio/listen/#{song_id}" + else + raise "unknown song ID" end end - if config_get("songs.sources.allow_transcoding") - if !config_get("songs.sources.proxy_downloads").as?(Bool) - raise "can't download a song with transcoding but without proxying allowed" - end + if DATABASE.scalar("select count(*) from song_data where id = ?", song_id).as(Int64) > 0 + song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_one("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ?", song_id, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?}) - canonical_url = metadata["webpage_url"].as_s? || metadata["original_url"].as_s? || url - - target_path = Path.new("data", "#{id}.mp3") - - Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", canonical_url], output: STDOUT, error: STDOUT) - - size = File.size(target_path) - - # todo: don't point to localhost - Song.new(metadata["fulltitle"].as_s? || metadata["title"].as_s? || "Song", metadata["uploader"].as_s? || "", size.to_i32, "http://localhost:8080/#{id}.mp3", canonical_url) + fetch_url = download_url + size = song_size + author_id = song_author_id + metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration) else - # todo + begin + metadata = fetch_song_metadata(url.not_nil!) + rescue err + puts "ran into error fetching metadata: #{err}; disabling song" + puts err.inspect + if song_exists + DATABASE.exec("update songs set disabled=1 where id = ?", song_id) + else + DATABASE.exec("insert into songs (id, url, disabled) values (?, ?, 1)", song_id, url) + end + else + if song_exists && url != metadata.normalized_url + DATABASE.exec("update songs set url = ? where id = ?", metadata.normalized_url, song_id) + end + + if DATABASE.scalar("select count(*) from songs join song_data on songs.id = song_data.id where songs.id != ? and url = ?", song_id, metadata.normalized_url).as(Int64) > 0 + # just use that song's metadata instead + # todo: dedup this and the above similar block somehow? + + song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_all("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ?", song_id, metadata.normalized_url, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})[0] + + fetch_url = download_url + size = song_size + author_id = song_author_id + metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration) + end + end end + + puts metadata.inspect + + # todo: insert into song_data + + # do checks to make sure this is a valid song + max_duration = config_get("songs.sources.max_duration").as?(Int64) + # todo + + if (fetch_url || !get_download) && metadata && size && author_id + # we're done! woo + if fetch_url && fetch_url.starts_with?("./") + # todo + fetch_url = "localhost:8080/#{fetch_url[2..]}" + end + return {metadata.name, author_id, metadata.author, size, fetch_url} + end + + metadata = metadata.not_nil! + + if get_download + if config_get("songs.sources.allow_transcoding") + if !config_get("songs.sources.proxy_downloads").as?(Bool) + raise "can't download a song with transcoding but without proxying allowed" + end + + # todo: check if song file exists + + target_path = get_file_path(song_id) + + Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", metadata.normalized_url], output: STDOUT, error: STDOUT) + + size = File.size(target_path).to_i + + fetch_url = "./#{song_id}.mp3" + else + # todo + raise "fetching songs without transcoding and proxying downloads currently unimplemented" + end + + # todo: update song_data with size, duration and url + end + + if !author_id + # todo + author_id = 1 + end + + if fetch_url && fetch_url.starts_with?("./") + # todo + # todo also: deduplicate this with similar block above? + fetch_url = "localhost:8080/#{fetch_url[2..]}" + end + return {metadata.name, author_id, metadata.author, size, fetch_url} end end