custom song system more-or-less outlined
it's functional, but missing a lot of the functionality; very little checks to match the config are done and caching is completely missing
This commit is contained in:
parent
6785d0e936
commit
8b3a39da9d
|
@ -2,5 +2,14 @@
|
||||||
"crystal-lang.completion": true,
|
"crystal-lang.completion": true,
|
||||||
"crystal-lang.hover": true,
|
"crystal-lang.hover": true,
|
||||||
"crystal-lang.implementations": true,
|
"crystal-lang.implementations": true,
|
||||||
"crystal-lang.mainFile": "${workspaceRoot}/src/crystal-gauntlet.cr"
|
"crystal-lang.mainFile": "${workspaceRoot}/src/crystal-gauntlet.cr",
|
||||||
|
"sqltools.connections": [
|
||||||
|
{
|
||||||
|
"previewLimit": 50,
|
||||||
|
"driver": "SQLite",
|
||||||
|
"name": "crystal-gauntlet",
|
||||||
|
"database": "${workspaceFolder:crystal-gauntlet}/crystalgauntlet.db"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"sqltools.useNodeRuntime": true
|
||||||
}
|
}
|
|
@ -1,13 +1,42 @@
|
||||||
-- +migrate up
|
-- +migrate up
|
||||||
CREATE TABLE songs (
|
CREATE TABLE songs (
|
||||||
id SERIAL PRIMARY KEY,
|
id SERIAL PRIMARY KEY,
|
||||||
name TEXT NOT NULL,
|
url TEXT NOT NULL,
|
||||||
author_id INTEGER NOT NULL,
|
disabled INTEGER NOT NULL DEFAULT 0,
|
||||||
author_name TEXT NOT NULL,
|
uploaded_by INTEGER references accounts(id)
|
||||||
size INTEGER NOT NULL, -- in bytes
|
|
||||||
download TEXT NOT NULL,
|
|
||||||
disabled INTEGER NOT NULL DEFAULT 0
|
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- song data is fetched on-demand rather than whenever songs are created,
|
||||||
|
-- so this is a seperate table that's filled in for any given song once
|
||||||
|
-- it's needed
|
||||||
|
CREATE TABLE song_data (
|
||||||
|
id SERIAL PRIMARY KEY references songs(id),
|
||||||
|
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
author_id INTEGER NOT NULL references song_authors(id),
|
||||||
|
|
||||||
|
source TEXT NOT NULL DEFAULT "unknown",
|
||||||
|
size INTEGER, -- in bytes
|
||||||
|
duration INTEGER,
|
||||||
|
|
||||||
|
-- this may contain an absolute url OR a relative one (starts with ./)
|
||||||
|
-- depending on if its local or remote
|
||||||
|
-- null indicates it should be re-fetched every time its queried
|
||||||
|
proxy_url TEXT,
|
||||||
|
|
||||||
|
last_updated TEXT NOT NULL DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE song_authors (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
source TEXT NOT NULL DEFAULT "unknown",
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
url TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO song_authors (id, name, source, url) VALUES (1, "", "unknown", "");
|
||||||
|
|
||||||
-- +migrate down
|
-- +migrate down
|
||||||
DROP TABLE songs;
|
DROP TABLE songs;
|
||||||
|
DROP TABLE song_data;
|
||||||
|
DROP TABLE song_authors;
|
|
@ -10,6 +10,8 @@ targets:
|
||||||
main: src/crystal-gauntlet.cr
|
main: src/crystal-gauntlet.cr
|
||||||
|
|
||||||
dependencies:
|
dependencies:
|
||||||
|
db:
|
||||||
|
github: crystal-lang/crystal-db
|
||||||
sqlite3:
|
sqlite3:
|
||||||
github: crystal-lang/crystal-sqlite3
|
github: crystal-lang/crystal-sqlite3
|
||||||
migrate:
|
migrate:
|
||||||
|
@ -20,8 +22,6 @@ dependencies:
|
||||||
toml:
|
toml:
|
||||||
github: crystal-community/toml.cr
|
github: crystal-community/toml.cr
|
||||||
branch: master
|
branch: master
|
||||||
crystagiri:
|
|
||||||
github: madeindjs/crystagiri
|
|
||||||
|
|
||||||
crystal: 1.6.2
|
crystal: 1.6.2
|
||||||
|
|
||||||
|
|
|
@ -46,20 +46,31 @@ module CrystalGauntlet
|
||||||
# expunge trailing slashes
|
# expunge trailing slashes
|
||||||
path = context.request.path.chomp("/")
|
path = context.request.path.chomp("/")
|
||||||
|
|
||||||
path = path.sub(config_get("general.append_path").as(String | Nil) || "", "")
|
# todo: rethink life choices
|
||||||
body = context.request.body
|
if path.ends_with?(".mp3")
|
||||||
|
# todo: BIG NONO
|
||||||
if !body
|
# todo: path traversal exploits SCARY
|
||||||
puts "no body :("
|
file = File.open("./data#{path}", "r")
|
||||||
elsif @@endpoints.has_key?(path)
|
context.response.content_type = "audio/mp3"
|
||||||
func = @@endpoints[path]
|
IO.copy(file, context.response)
|
||||||
value = func.call(body.gets_to_end)
|
file.close
|
||||||
context.response.content_type = "text/plain"
|
|
||||||
context.response.print value
|
|
||||||
puts "#{path} -> #{value}"
|
|
||||||
else
|
else
|
||||||
context.response.respond_with_status(404, "endpoint not found")
|
path = path.sub(config_get("general.append_path").as(String | Nil) || "", "")
|
||||||
puts "#{path} -> 404"
|
|
||||||
|
body = context.request.body
|
||||||
|
|
||||||
|
if !body
|
||||||
|
puts "no body :("
|
||||||
|
elsif @@endpoints.has_key?(path)
|
||||||
|
func = @@endpoints[path]
|
||||||
|
value = func.call(body.gets_to_end)
|
||||||
|
context.response.content_type = "text/plain"
|
||||||
|
context.response.print value
|
||||||
|
puts "#{path} -> #{value}"
|
||||||
|
else
|
||||||
|
context.response.respond_with_status(404, "endpoint not found")
|
||||||
|
puts "#{path} -> 404"
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -132,7 +132,8 @@ CrystalGauntlet.endpoints["/getGJLevels21.php"] = ->(body : String): String {
|
||||||
# todo: search query
|
# todo: search query
|
||||||
|
|
||||||
where_str = "where (#{queryParams.join(") and (")})"
|
where_str = "where (#{queryParams.join(") and (")})"
|
||||||
query_base = "from levels join users on levels.user_id = users.id left join songs on levels.song_id = songs.id left join map_pack_links on map_pack_links.level_id = levels.id #{where_str} order by #{order}"
|
# todo: switch join users to left join to avoid losing levels to the shadow realm after a user vanishes
|
||||||
|
query_base = "from levels join users on levels.user_id = users.id left join map_pack_links on map_pack_links.level_id = levels.id #{where_str} order by #{order}"
|
||||||
|
|
||||||
puts query_base
|
puts query_base
|
||||||
|
|
||||||
|
@ -144,99 +145,71 @@ CrystalGauntlet.endpoints["/getGJLevels21.php"] = ->(body : String): String {
|
||||||
|
|
||||||
hash_data = [] of Tuple(Int32, Int32, Bool)
|
hash_data = [] of Tuple(Int32, Int32, Bool)
|
||||||
|
|
||||||
DATABASE.query "select levels.id, levels.name, levels.user_id, levels.description, levels.original, levels.game_version, levels.requested_stars, levels.version, levels.song_id, levels.length, levels.objects, levels.coins, levels.has_ldm, levels.two_player, levels.downloads, levels.likes, levels.difficulty, levels.community_difficulty, levels.demon_difficulty, levels.stars, levels.featured, levels.epic, levels.rated_coins, users.username, users.udid, users.account_id, users.registered, songs.name, songs.author_id, songs.author_name, songs.size, songs.disabled, songs.download #{query_base} limit #{levels_per_page} offset #{page_offset}" do |rs|
|
# fucking help
|
||||||
rs.each do
|
DATABASE.query_all("select levels.id, levels.name, levels.user_id, levels.description, levels.original, levels.game_version, levels.requested_stars, levels.version, levels.song_id, levels.length, levels.objects, levels.coins, levels.has_ldm, levels.two_player, levels.downloads, levels.likes, levels.difficulty, levels.community_difficulty, levels.demon_difficulty, levels.stars, levels.featured, levels.epic, levels.rated_coins, users.username, users.udid, users.account_id, users.registered #{query_base} limit #{levels_per_page} offset #{page_offset}", as: {Int32, String, Int32, String, Int32 | Nil, Int32, Int32 | Nil, Int32, Int32, Int32, Int32, Int32, Bool, Bool, Int32, Int32, Int32 | Nil, Int32 | Nil, Int32 | Nil, Int32 | Nil, Bool, Bool, Bool, String, String | Nil, Int32 | Nil, Bool}).map() do |id, name, user_id, description, original, game_version, requested_stars, version, song_id, length, objects, coins, has_ldm, two_player, downloads, likes, set_difficulty_int, community_difficulty_int, demon_difficulty_int, stars, featured, epic, rated_coins, user_username, user_udid, user_account_id, user_registered|
|
||||||
id = rs.read(Int32)
|
set_difficulty = set_difficulty_int && LevelDifficulty.new(set_difficulty_int)
|
||||||
name = rs.read(String)
|
community_difficulty = community_difficulty_int && LevelDifficulty.new(community_difficulty_int)
|
||||||
user_id = rs.read(Int32)
|
difficulty = set_difficulty || community_difficulty
|
||||||
description = rs.read(String)
|
demon_difficulty = demon_difficulty_int && DemonDifficulty.new(demon_difficulty_int)
|
||||||
original = rs.read(Int32 | Nil)
|
|
||||||
game_version = rs.read(Int32)
|
|
||||||
requested_stars = rs.read(Int32 | Nil)
|
|
||||||
version = rs.read(Int32)
|
|
||||||
song_id = rs.read(Int32)
|
|
||||||
length = rs.read(Int32)
|
|
||||||
objects = rs.read(Int32)
|
|
||||||
coins = rs.read(Int32)
|
|
||||||
has_ldm = rs.read(Bool)
|
|
||||||
two_player = rs.read(Bool)
|
|
||||||
downloads = rs.read(Int32)
|
|
||||||
likes = rs.read(Int32)
|
|
||||||
set_difficulty_int = rs.read(Int32 | Nil)
|
|
||||||
set_difficulty = set_difficulty_int && LevelDifficulty.new(set_difficulty_int)
|
|
||||||
community_difficulty_int = rs.read(Int32 | Nil)
|
|
||||||
community_difficulty = community_difficulty_int && LevelDifficulty.new(community_difficulty_int)
|
|
||||||
difficulty = set_difficulty || community_difficulty
|
|
||||||
demon_difficulty_int = rs.read(Int32 | Nil)
|
|
||||||
demon_difficulty = demon_difficulty_int && DemonDifficulty.new(demon_difficulty_int)
|
|
||||||
stars = rs.read(Int32 | Nil)
|
|
||||||
featured = rs.read(Bool)
|
|
||||||
epic = rs.read(Bool)
|
|
||||||
rated_coins = rs.read(Bool)
|
|
||||||
|
|
||||||
user_username = rs.read(String)
|
# https://github.com/Cvolton/GMDprivateServer/blob/master/incl/levels/getGJLevels.php#L266
|
||||||
user_udid = rs.read(String | Nil)
|
results << Format.fmt_hash({
|
||||||
user_account_id = rs.read(Int32 | Nil)
|
1 => id,
|
||||||
user_registered = rs.read(Bool)
|
2 => name,
|
||||||
|
5 => version,
|
||||||
|
6 => user_id,
|
||||||
|
8 => 10,
|
||||||
|
9 => difficulty ? difficulty.to_star_difficulty : 0, # 0=N/A 10=EASY 20=NORMAL 30=HARD 40=HARDER 50=INSANE 50=AUTO 50=DEMON
|
||||||
|
10 => downloads,
|
||||||
|
12 => !Songs.is_custom_song(song_id) ? song_id : 0,
|
||||||
|
13 => game_version,
|
||||||
|
14 => likes,
|
||||||
|
17 => difficulty && difficulty.demon?,
|
||||||
|
# 0 for n/a, 10 for easy, 20, for medium, ...
|
||||||
|
43 => (demon_difficulty || DemonDifficulty::Hard).to_demon_difficulty,
|
||||||
|
25 => difficulty && difficulty.auto?,
|
||||||
|
18 => stars || 0,
|
||||||
|
19 => featured,
|
||||||
|
42 => epic,
|
||||||
|
45 => objects,
|
||||||
|
3 => GDBase64.encode(description),
|
||||||
|
15 => length,
|
||||||
|
30 => original || 0,
|
||||||
|
31 => two_player,
|
||||||
|
37 => coins,
|
||||||
|
38 => rated_coins,
|
||||||
|
39 => requested_stars || 0,
|
||||||
|
46 => 1,
|
||||||
|
47 => 2,
|
||||||
|
40 => has_ldm,
|
||||||
|
35 => Songs.is_custom_song(song_id) ? song_id : 0,
|
||||||
|
})
|
||||||
|
|
||||||
song_name = rs.read(String | Nil)
|
users << "#{user_id}:#{user_username}:#{user_registered ? user_account_id : user_udid}"
|
||||||
song_author_id = rs.read(Int32 | Nil)
|
|
||||||
song_author_name = rs.read(String | Nil)
|
|
||||||
song_size = rs.read(Int32 | Nil)
|
|
||||||
song_disabled = rs.read(Int32 | Nil)
|
|
||||||
song_download = rs.read(String | Nil)
|
|
||||||
|
|
||||||
# https://github.com/Cvolton/GMDprivateServer/blob/master/incl/levels/getGJLevels.php#L266
|
if Songs.is_custom_song(song_id)
|
||||||
results << Format.fmt_hash({
|
begin
|
||||||
1 => id,
|
song = Songs.fetch_song(song_id, false)
|
||||||
2 => name,
|
rescue
|
||||||
5 => version,
|
else
|
||||||
6 => user_id,
|
if song != nil
|
||||||
8 => 10,
|
song_name, song_author_id, song_author_name, song_size, song_download = song.not_nil!
|
||||||
9 => difficulty ? difficulty.to_star_difficulty : 0, # 0=N/A 10=EASY 20=NORMAL 30=HARD 40=HARDER 50=INSANE 50=AUTO 50=DEMON
|
songs << Format.fmt_song({
|
||||||
10 => downloads,
|
1 => song_id,
|
||||||
12 => !Songs.is_custom_song(song_id) ? song_id : 0,
|
2 => song_name,
|
||||||
13 => game_version,
|
3 => song_author_id,
|
||||||
14 => likes,
|
4 => song_author_name,
|
||||||
17 => difficulty && difficulty.demon?,
|
5 => (song_size || 0) / (1000 * 1000),
|
||||||
# 0 for n/a, 10 for easy, 20, for medium, ...
|
6 => "",
|
||||||
43 => (demon_difficulty || DemonDifficulty::Hard).to_demon_difficulty,
|
10 => song_download || "",
|
||||||
25 => difficulty && difficulty.auto?,
|
7 => "",
|
||||||
18 => stars || 0,
|
8 => "1"
|
||||||
19 => featured,
|
})
|
||||||
42 => epic,
|
end
|
||||||
45 => objects,
|
|
||||||
3 => GDBase64.encode(description),
|
|
||||||
15 => length,
|
|
||||||
30 => original || 0,
|
|
||||||
31 => two_player,
|
|
||||||
37 => coins,
|
|
||||||
38 => rated_coins,
|
|
||||||
39 => requested_stars || 0,
|
|
||||||
46 => 1,
|
|
||||||
47 => 2,
|
|
||||||
40 => has_ldm,
|
|
||||||
35 => Songs.is_custom_song(song_id) ? song_id : 0,
|
|
||||||
})
|
|
||||||
|
|
||||||
users << "#{user_id}:#{user_username}:#{user_registered ? user_account_id : user_udid}"
|
|
||||||
|
|
||||||
if Songs.is_custom_song(song_id) && song_disabled == 0
|
|
||||||
songs << Format.fmt_song({
|
|
||||||
1 => song_id,
|
|
||||||
2 => song_name,
|
|
||||||
3 => song_author_id,
|
|
||||||
4 => song_author_name,
|
|
||||||
5 => song_size.not_nil! / (1000 * 1000),
|
|
||||||
6 => "",
|
|
||||||
10 => song_download,
|
|
||||||
7 => "",
|
|
||||||
8 => "1"
|
|
||||||
})
|
|
||||||
end
|
end
|
||||||
|
|
||||||
hash_data << {id, stars || 0, rated_coins}
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
hash_data << {id, stars || 0, rated_coins}
|
||||||
end
|
end
|
||||||
|
|
||||||
# `${amount}:${offset}:${levelsPerPage}`
|
# `${amount}:${offset}:${levelsPerPage}`
|
||||||
|
|
|
@ -1,5 +1,4 @@
|
||||||
require "uri"
|
require "uri"
|
||||||
require "crystagiri"
|
|
||||||
require "http/client"
|
require "http/client"
|
||||||
require "digest/sha256"
|
require "digest/sha256"
|
||||||
|
|
||||||
|
@ -17,71 +16,27 @@ CrystalGauntlet.endpoints["/getGJSongInfo.php"] = ->(body : String): String {
|
||||||
|
|
||||||
song_id = params["songID"].to_i32
|
song_id = params["songID"].to_i32
|
||||||
|
|
||||||
DATABASE.query("select name, author_id, author_name, size, download, disabled from songs where id = ?", song_id) do |rs|
|
song = Songs.fetch_song(song_id, true)
|
||||||
if rs.move_next
|
|
||||||
song_name = rs.read(String)
|
|
||||||
author_id = rs.read(Int32)
|
|
||||||
author_name = rs.read(String)
|
|
||||||
size = rs.read(Int32)
|
|
||||||
download = rs.read(String)
|
|
||||||
disabled = rs.read(Int32)
|
|
||||||
|
|
||||||
if disabled == 1
|
|
||||||
return "-2"
|
|
||||||
end
|
|
||||||
|
|
||||||
|
if song != nil
|
||||||
|
begin
|
||||||
|
song_name, song_author_id, song_author_name, song_size, song_download = song.not_nil!
|
||||||
|
rescue
|
||||||
|
return "-1"
|
||||||
|
else
|
||||||
return Format.fmt_song({
|
return Format.fmt_song({
|
||||||
1 => song_id,
|
1 => song_id,
|
||||||
2 => song_name,
|
2 => song_name,
|
||||||
3 => author_id,
|
3 => song_author_id,
|
||||||
4 => author_name,
|
4 => song_author_name,
|
||||||
5 => size / (1000 * 1000),
|
5 => (song_size || 0) / (1000 * 1000),
|
||||||
6 => "",
|
6 => "",
|
||||||
10 => download,
|
10 => song_download || "",
|
||||||
7 => "",
|
7 => "",
|
||||||
8 => "0"
|
8 => "1"
|
||||||
})
|
})
|
||||||
end
|
end
|
||||||
end
|
|
||||||
|
|
||||||
if Songs.is_reuploaded_song(song_id)
|
|
||||||
# todo
|
|
||||||
"-1"
|
|
||||||
else
|
else
|
||||||
# todo: maybe use yt-dlp? for other sources too
|
return "-2"
|
||||||
doc = Crystagiri::HTML.from_url "https://www.newgrounds.com/audio/listen/#{song_id}"
|
|
||||||
|
|
||||||
song_name = (doc.css("title") { |d| })[0].content
|
|
||||||
song_artist = (doc.css(".item-details-main > h4 > a") { |d| d })[0].content
|
|
||||||
song_url_str = (doc.css("script") { |d| })
|
|
||||||
.map { |d| d.node.to_s.match(NEWGROUNDS_AUDIO_URL_REGEX) }
|
|
||||||
.reduce { |acc, d| acc || d }
|
|
||||||
.not_nil![1]
|
|
||||||
|
|
||||||
# todo: proxy locally
|
|
||||||
song_url = unescape_string(song_url_str).split("?")[0].sub("https://", "http://")
|
|
||||||
|
|
||||||
# todo: consider hashes?
|
|
||||||
size = 0
|
|
||||||
|
|
||||||
HTTP::Client.head(song_url) do |response|
|
|
||||||
size = response.headers["content-length"].to_i
|
|
||||||
end
|
|
||||||
|
|
||||||
author_id = 9 # todo: what is this needed for?
|
|
||||||
|
|
||||||
DATABASE.exec("insert into songs (id, name, author_id, author_name, size, download) values (?, ?, ?, ?, ?, ?)", song_id, song_name, author_id, song_artist, size, song_url)
|
|
||||||
|
|
||||||
return Format.fmt_song({
|
|
||||||
1 => song_id,
|
|
||||||
2 => song_name,
|
|
||||||
3 => author_id,
|
|
||||||
4 => song_artist,
|
|
||||||
5 => size / (1000 * 1000),
|
|
||||||
6 => "",
|
|
||||||
10 => song_url,
|
|
||||||
7 => "",
|
|
||||||
8 => "0"
|
|
||||||
})
|
|
||||||
end
|
end
|
||||||
}
|
}
|
||||||
|
|
205
src/lib/songs.cr
205
src/lib/songs.cr
|
@ -1,4 +1,5 @@
|
||||||
require "json"
|
require "json"
|
||||||
|
require "db"
|
||||||
|
|
||||||
include CrystalGauntlet
|
include CrystalGauntlet
|
||||||
|
|
||||||
|
@ -7,20 +8,49 @@ module CrystalGauntlet::Songs
|
||||||
|
|
||||||
GD_AUDIO_FORMAT = "mp3"
|
GD_AUDIO_FORMAT = "mp3"
|
||||||
|
|
||||||
|
# todo: make this configurable
|
||||||
|
REUPLOADED_SONG_ADD_ID = 5000000
|
||||||
|
|
||||||
|
CUSTOM_SONG_START = 50
|
||||||
|
|
||||||
|
# set in 6_songs.sql
|
||||||
|
UNKNOWN_SONG_AUTHOR = 1
|
||||||
|
|
||||||
def is_custom_song(id)
|
def is_custom_song(id)
|
||||||
id >= 50
|
id >= CUSTOM_SONG_START
|
||||||
end
|
end
|
||||||
|
|
||||||
def is_reuploaded_song(id)
|
def is_reuploaded_song(id)
|
||||||
id >= 5000000
|
id >= REUPLOADED_SONG_ADD_ID
|
||||||
end
|
end
|
||||||
|
|
||||||
class Song
|
class SongMetadata
|
||||||
def initialize(name : String, author : String, size : Int32, download_url : String | Nil, normalized_url : String)
|
def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil)
|
||||||
@name = name
|
@name = name
|
||||||
@author = author
|
@author = author
|
||||||
@size = size
|
@normalized_url = normalized_url
|
||||||
@download_url = download_url
|
@source = source
|
||||||
|
@author_url = author_url
|
||||||
|
@duration = duration
|
||||||
|
end
|
||||||
|
|
||||||
|
def name
|
||||||
|
@name
|
||||||
|
end
|
||||||
|
def author
|
||||||
|
@author
|
||||||
|
end
|
||||||
|
def normalized_url
|
||||||
|
@normalized_url
|
||||||
|
end
|
||||||
|
def source
|
||||||
|
@source
|
||||||
|
end
|
||||||
|
def author_url
|
||||||
|
@author_url
|
||||||
|
end
|
||||||
|
def duration
|
||||||
|
@duration
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -28,49 +58,158 @@ module CrystalGauntlet::Songs
|
||||||
config_get("songs.allow_all_sources").as?(Bool) || config_get("songs.sources.#{source}.allow").as?(Bool) || false
|
config_get("songs.allow_all_sources").as?(Bool) || config_get("songs.sources.#{source}.allow").as?(Bool) || false
|
||||||
end
|
end
|
||||||
|
|
||||||
def reupload(url : String, id : Int32) : Song | Nil
|
def get_file_path(song_id : Int32)
|
||||||
puts url
|
Path.new("data", "#{song_id}.mp3")
|
||||||
|
end
|
||||||
|
|
||||||
|
# will raise errors
|
||||||
|
def fetch_song_metadata(url : String) : SongMetadata
|
||||||
|
puts "getting metadata for #{url}"
|
||||||
|
|
||||||
output = IO::Memory.new
|
output = IO::Memory.new
|
||||||
# todo: ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ LOOK OUT FOR SHELL INJECTION BULLSHIT!!!!!!!!!!!!!!!!!!
|
# todo: ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ LOOK OUT FOR SHELL INJECTION BULLSHIT!!!!!!!!!!!!!!!!!!
|
||||||
Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-J", url], output: output)
|
Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-J", url], output: output)
|
||||||
output.close
|
output.close
|
||||||
|
|
||||||
puts output.to_s
|
|
||||||
|
|
||||||
metadata = JSON.parse(output.to_s)
|
metadata = JSON.parse(output.to_s)
|
||||||
|
|
||||||
if !is_source_allowed(metadata["extractor"].as_s? || "unknown")
|
canonical_url = metadata["webpage_url"].as_s? || metadata["original_url"].as_s? || url
|
||||||
raise "source forbidden: #{metadata["extractor"]}"
|
duration = metadata["duration"]? && (metadata["duration"].as_f? || metadata["duration"].as_i?)
|
||||||
|
|
||||||
|
return SongMetadata.new(
|
||||||
|
(metadata["fulltitle"]? && metadata["fulltitle"].as_s?) || (metadata["title"]? && metadata["title"].as_s?) || url,
|
||||||
|
(metadata["uploader"]? && metadata["uploader"].as_s?) || "",
|
||||||
|
canonical_url,
|
||||||
|
metadata["extractor"].as_s,
|
||||||
|
(metadata.["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url,
|
||||||
|
duration ? duration.to_i : nil
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
# name, author id, author name, size, download url
|
||||||
|
# returns nil if song should be disabled
|
||||||
|
# throws if something failed
|
||||||
|
def fetch_song(song_id : Int32, get_download = false) : Tuple(String, Int32, String, Int32 | Nil, String | Nil) | Nil
|
||||||
|
puts "fetching #{song_id}"
|
||||||
|
if !config_get("songs.allow_custom_songs").as?(Bool)
|
||||||
|
puts "custom songs not allowed"
|
||||||
|
return nil
|
||||||
end
|
end
|
||||||
|
|
||||||
max_duration = config_get("songs.sources.max_duration").as?(Int64) || 0
|
# todo: this is kinda spaghetti
|
||||||
|
metadata = nil
|
||||||
|
author_id = nil
|
||||||
|
size = nil
|
||||||
|
fetch_url = nil
|
||||||
|
|
||||||
if max_duration > 0
|
song_exists = false
|
||||||
if !metadata["duration"]
|
url = nil
|
||||||
raise "failed to determine track duration"
|
|
||||||
elsif metadata["duration"].as_f >= max_duration
|
begin
|
||||||
raise "track goes above max track duration (#{max_duration}s)"
|
url, disabled = DATABASE.query_one("select url, disabled from songs where id = ?", song_id, as: {String, Bool})
|
||||||
|
|
||||||
|
if disabled
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
|
||||||
|
song_exists = true
|
||||||
|
rescue
|
||||||
|
if config_get("songs.preserve_newgrounds_ids").as?(Bool)
|
||||||
|
url = "https://www.newgrounds.com/audio/listen/#{song_id}"
|
||||||
|
else
|
||||||
|
raise "unknown song ID"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
if config_get("songs.sources.allow_transcoding")
|
if DATABASE.scalar("select count(*) from song_data where id = ?", song_id).as(Int64) > 0
|
||||||
if !config_get("songs.sources.proxy_downloads").as?(Bool)
|
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_one("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ?", song_id, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})
|
||||||
raise "can't download a song with transcoding but without proxying allowed"
|
|
||||||
end
|
|
||||||
|
|
||||||
canonical_url = metadata["webpage_url"].as_s? || metadata["original_url"].as_s? || url
|
fetch_url = download_url
|
||||||
|
size = song_size
|
||||||
target_path = Path.new("data", "#{id}.mp3")
|
author_id = song_author_id
|
||||||
|
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration)
|
||||||
Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", canonical_url], output: STDOUT, error: STDOUT)
|
|
||||||
|
|
||||||
size = File.size(target_path)
|
|
||||||
|
|
||||||
# todo: don't point to localhost
|
|
||||||
Song.new(metadata["fulltitle"].as_s? || metadata["title"].as_s? || "Song", metadata["uploader"].as_s? || "", size.to_i32, "http://localhost:8080/#{id}.mp3", canonical_url)
|
|
||||||
else
|
else
|
||||||
# todo
|
begin
|
||||||
|
metadata = fetch_song_metadata(url.not_nil!)
|
||||||
|
rescue err
|
||||||
|
puts "ran into error fetching metadata: #{err}; disabling song"
|
||||||
|
puts err.inspect
|
||||||
|
if song_exists
|
||||||
|
DATABASE.exec("update songs set disabled=1 where id = ?", song_id)
|
||||||
|
else
|
||||||
|
DATABASE.exec("insert into songs (id, url, disabled) values (?, ?, 1)", song_id, url)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
if song_exists && url != metadata.normalized_url
|
||||||
|
DATABASE.exec("update songs set url = ? where id = ?", metadata.normalized_url, song_id)
|
||||||
|
end
|
||||||
|
|
||||||
|
if DATABASE.scalar("select count(*) from songs join song_data on songs.id = song_data.id where songs.id != ? and url = ?", song_id, metadata.normalized_url).as(Int64) > 0
|
||||||
|
# just use that song's metadata instead
|
||||||
|
# todo: dedup this and the above similar block somehow?
|
||||||
|
|
||||||
|
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_all("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ?", song_id, metadata.normalized_url, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})[0]
|
||||||
|
|
||||||
|
fetch_url = download_url
|
||||||
|
size = song_size
|
||||||
|
author_id = song_author_id
|
||||||
|
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration)
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
puts metadata.inspect
|
||||||
|
|
||||||
|
# todo: insert into song_data
|
||||||
|
|
||||||
|
# do checks to make sure this is a valid song
|
||||||
|
max_duration = config_get("songs.sources.max_duration").as?(Int64)
|
||||||
|
# todo
|
||||||
|
|
||||||
|
if (fetch_url || !get_download) && metadata && size && author_id
|
||||||
|
# we're done! woo
|
||||||
|
if fetch_url && fetch_url.starts_with?("./")
|
||||||
|
# todo
|
||||||
|
fetch_url = "localhost:8080/#{fetch_url[2..]}"
|
||||||
|
end
|
||||||
|
return {metadata.name, author_id, metadata.author, size, fetch_url}
|
||||||
|
end
|
||||||
|
|
||||||
|
metadata = metadata.not_nil!
|
||||||
|
|
||||||
|
if get_download
|
||||||
|
if config_get("songs.sources.allow_transcoding")
|
||||||
|
if !config_get("songs.sources.proxy_downloads").as?(Bool)
|
||||||
|
raise "can't download a song with transcoding but without proxying allowed"
|
||||||
|
end
|
||||||
|
|
||||||
|
# todo: check if song file exists
|
||||||
|
|
||||||
|
target_path = get_file_path(song_id)
|
||||||
|
|
||||||
|
Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", metadata.normalized_url], output: STDOUT, error: STDOUT)
|
||||||
|
|
||||||
|
size = File.size(target_path).to_i
|
||||||
|
|
||||||
|
fetch_url = "./#{song_id}.mp3"
|
||||||
|
else
|
||||||
|
# todo
|
||||||
|
raise "fetching songs without transcoding and proxying downloads currently unimplemented"
|
||||||
|
end
|
||||||
|
|
||||||
|
# todo: update song_data with size, duration and url
|
||||||
|
end
|
||||||
|
|
||||||
|
if !author_id
|
||||||
|
# todo
|
||||||
|
author_id = 1
|
||||||
|
end
|
||||||
|
|
||||||
|
if fetch_url && fetch_url.starts_with?("./")
|
||||||
|
# todo
|
||||||
|
# todo also: deduplicate this with similar block above?
|
||||||
|
fetch_url = "localhost:8080/#{fetch_url[2..]}"
|
||||||
|
end
|
||||||
|
return {metadata.name, author_id, metadata.author, size, fetch_url}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue