246 lines
8.8 KiB
Crystal
246 lines
8.8 KiB
Crystal
require "json"
|
|
require "db"
|
|
|
|
include CrystalGauntlet
|
|
|
|
module CrystalGauntlet::Songs
|
|
extend self
|
|
|
|
GD_AUDIO_FORMAT = "mp3"
|
|
|
|
# todo: make this configurable
|
|
REUPLOADED_SONG_ADD_ID = 5000000
|
|
|
|
CUSTOM_SONG_START = 50
|
|
|
|
# set in 6_songs.sql
|
|
UNKNOWN_SONG_AUTHOR = 1
|
|
|
|
def is_custom_song(id)
|
|
id >= CUSTOM_SONG_START
|
|
end
|
|
|
|
def is_reuploaded_song(id)
|
|
id >= REUPLOADED_SONG_ADD_ID
|
|
end
|
|
|
|
class SongMetadata
|
|
def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil, size : Int32 | Nil)
|
|
@name = name
|
|
@author = author
|
|
@normalized_url = normalized_url
|
|
@source = source
|
|
@author_url = author_url
|
|
@duration = duration
|
|
@size = size
|
|
end
|
|
|
|
def name
|
|
@name
|
|
end
|
|
def author
|
|
@author
|
|
end
|
|
def normalized_url
|
|
@normalized_url
|
|
end
|
|
def source
|
|
@source
|
|
end
|
|
def author_url
|
|
@author_url
|
|
end
|
|
def duration
|
|
@duration
|
|
end
|
|
def size
|
|
@size
|
|
end
|
|
end
|
|
|
|
def is_source_allowed(source : String) : Bool
|
|
config_get("songs.allow_all_sources").as?(Bool) || config_get("songs.sources.#{source}.allow").as?(Bool) || false
|
|
end
|
|
|
|
def get_file_path(song_id : Int32)
|
|
DATA_FOLDER / "songs" / "#{song_id}.mp3"
|
|
end
|
|
|
|
# will raise errors
|
|
def fetch_song_metadata(url : String) : SongMetadata
|
|
LOG.info { "getting metadata for #{url}" }
|
|
|
|
output = IO::Memory.new
|
|
# todo: ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ LOOK OUT FOR SHELL INJECTION BULLSHIT!!!!!!!!!!!!!!!!!!
|
|
Process.run(config_get("songs.sources.ytdlp_binary", "yt-dlp"), ["-J", url], output: output)
|
|
output.close
|
|
|
|
metadata = JSON.parse(output.to_s)
|
|
|
|
canonical_url = metadata["webpage_url"].as_s? || metadata["original_url"].as_s? || url
|
|
duration = metadata["duration"]? && (metadata["duration"].as_f? || metadata["duration"].as_i?)
|
|
|
|
return SongMetadata.new(
|
|
(metadata["fulltitle"]? && metadata["fulltitle"].as_s?) || (metadata["title"]? && metadata["title"].as_s?) || url,
|
|
(metadata["uploader"]? && metadata["uploader"].as_s?) || "",
|
|
canonical_url,
|
|
metadata["extractor"].as_s,
|
|
(metadata["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url,
|
|
duration ? duration.to_i : nil,
|
|
metadata["filesize"]? && metadata["filesize"].as_i?
|
|
)
|
|
end
|
|
|
|
def get_artist_id(artist_name : String, artist_url : String, source : String) : Int32
|
|
if source == "unknown"
|
|
return UNKNOWN_SONG_AUTHOR
|
|
end
|
|
if artist_name.blank?
|
|
return UNKNOWN_SONG_AUTHOR
|
|
end
|
|
|
|
begin
|
|
DATABASE.query_one("select id from song_authors where name = ? and url = ? and source = ?", artist_name, artist_url, source, as: {Int32})
|
|
rescue
|
|
next_id = IDs.get_next_id("song_authors")
|
|
DATABASE.exec("insert into song_authors (id, source, name, url) values (?, ?, ?, ?)", next_id, source, artist_name, artist_url)
|
|
next_id.to_i
|
|
end
|
|
end
|
|
|
|
# name, author id, author name, size, download url
|
|
# returns nil if song should be disabled
|
|
# throws if something failed
|
|
# todo: why is size sometimes 0.00 despite the song being downloaded?
|
|
def fetch_song(song_id : Int32, get_download = false) : Tuple(String, Int32, String, Int32 | Nil, String | Nil) | Nil
|
|
LOG.debug { "fetching #{song_id}" }
|
|
if !config_get("songs.allow_custom_songs").as?(Bool)
|
|
return nil
|
|
end
|
|
|
|
# todo: this is kinda spaghetti
|
|
metadata = nil
|
|
author_id = nil
|
|
fetch_url = nil
|
|
|
|
song_exists = false
|
|
url = nil
|
|
|
|
begin
|
|
url, disabled = DATABASE.query_one("select url, disabled from songs where id = ?", song_id, as: {String, Bool})
|
|
|
|
if disabled
|
|
return nil
|
|
end
|
|
|
|
song_exists = true
|
|
rescue
|
|
if config_get("songs.preserve_newgrounds_ids").as?(Bool)
|
|
url = "https://www.newgrounds.com/audio/listen/#{song_id}"
|
|
else
|
|
raise "unknown song ID"
|
|
end
|
|
end
|
|
|
|
if DATABASE.scalar("select count(*) from song_data where id = ?", song_id).as(Int64) > 0
|
|
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_one("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ?", song_id, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})
|
|
|
|
fetch_url = download_url
|
|
author_id = song_author_id
|
|
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size)
|
|
else
|
|
if url == nil
|
|
# prevent disabling unoccupied IDs
|
|
return nil
|
|
end
|
|
begin
|
|
metadata = fetch_song_metadata(url.not_nil!)
|
|
rescue err
|
|
LOG.warn { "ran into error fetching metadata: #{err}; disabling song" }
|
|
LOG.warn { err.inspect }
|
|
if song_exists
|
|
DATABASE.exec("update songs set disabled=1 where id = ?", song_id)
|
|
else
|
|
DATABASE.exec("insert into songs (id, url, disabled) values (?, ?, 1)", song_id, url)
|
|
end
|
|
return nil
|
|
else
|
|
if song_exists && url != metadata.normalized_url
|
|
DATABASE.exec("update songs set url = ? where id = ?", metadata.normalized_url, song_id)
|
|
end
|
|
|
|
if DATABASE.scalar("select count(*) from songs join song_data on songs.id = song_data.id where songs.id != ? and url = ?", song_id, metadata.normalized_url).as(Int64) > 0
|
|
# just use that song's metadata instead
|
|
# todo: dedup this and the above similar block somehow?
|
|
|
|
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_all("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ?", song_id, metadata.normalized_url, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})[0]
|
|
|
|
fetch_url = download_url
|
|
author_id = song_author_id
|
|
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size)
|
|
end
|
|
end
|
|
end
|
|
|
|
LOG.debug { metadata.inspect }
|
|
|
|
# do checks to make sure this is a valid song
|
|
max_duration = config_get("songs.sources.max_duration").as?(Int64)
|
|
# todo
|
|
|
|
if (fetch_url || !get_download) && metadata && author_id
|
|
# we're done! woo
|
|
if fetch_url && fetch_url.starts_with?("./")
|
|
fetch_url = "#{config_get("general.hostname", "")}/#{fetch_url[2..]}"
|
|
end
|
|
return {metadata.name, author_id, metadata.author, metadata.size, fetch_url}
|
|
end
|
|
|
|
metadata = metadata.not_nil!
|
|
new_size = nil
|
|
|
|
if get_download
|
|
if config_get("songs.sources.allow_transcoding")
|
|
if !config_get("songs.sources.proxy_downloads").as?(Bool)
|
|
raise "can't download a song with transcoding but without proxying allowed"
|
|
end
|
|
|
|
# todo: check if song file exists
|
|
|
|
target_path = get_file_path(song_id)
|
|
|
|
Process.run(config_get("songs.sources.ytdlp_binary", "yt-dlp"), ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary", "ffmpeg"), metadata.normalized_url], output: STDOUT, error: STDOUT)
|
|
|
|
new_size = File.size(target_path).to_i
|
|
|
|
# todo: get duration
|
|
|
|
fetch_url = "./#{song_id}.mp3"
|
|
else
|
|
# todo
|
|
raise "fetching songs without transcoding and proxying downloads currently unimplemented"
|
|
end
|
|
end
|
|
|
|
if !author_id
|
|
author_id = get_artist_id(metadata.author, metadata.source, metadata.author_url)
|
|
end
|
|
|
|
if config_get("songs.sources.proxy_downloads")
|
|
if DATABASE.scalar("select count(*) from song_data where id = ?", song_id).as(Int64) > 0
|
|
DATABASE.exec("update song_data set name = ?, author_id = ?, size = ? where id = ?", metadata.name, author_id, new_size || metadata.size, song_id)
|
|
else
|
|
DATABASE.exec("insert into song_data (id, name, author_id, source, size, duration, proxy_url) values (?, ?, ?, ?, ?, ?, ?)", song_id, metadata.name, author_id, metadata.source, metadata.size, metadata.duration, fetch_url)
|
|
end
|
|
else
|
|
# todo
|
|
end
|
|
|
|
if fetch_url && fetch_url.starts_with?("./")
|
|
# todo: deduplicate this with similar block above?
|
|
fetch_url = "#{config_get("general.hostname").as?(String) || ""}/#{fetch_url[2..]}"
|
|
end
|
|
return {metadata.name, author_id, metadata.author, new_size || metadata.size, fetch_url}
|
|
end
|
|
end
|