2023-01-01 06:42:34 +01:00
require " json "
2023-01-02 07:51:41 +01:00
require " db "
2023-01-01 06:42:34 +01:00
2022-12-31 14:25:43 +01:00
include CrystalGauntlet
module CrystalGauntlet::Songs
extend self
2023-01-01 06:42:34 +01:00
GD_AUDIO_FORMAT = " mp3 "
2023-01-02 07:51:41 +01:00
# todo: make this configurable
REUPLOADED_SONG_ADD_ID = 5000000
CUSTOM_SONG_START = 50
# set in 6_songs.sql
UNKNOWN_SONG_AUTHOR = 1
2022-12-31 14:25:43 +01:00
def is_custom_song ( id )
2023-01-02 07:51:41 +01:00
id >= CUSTOM_SONG_START
2022-12-31 14:25:43 +01:00
end
def is_reuploaded_song ( id )
2023-01-02 07:51:41 +01:00
id >= REUPLOADED_SONG_ADD_ID
2022-12-31 14:25:43 +01:00
end
2023-01-01 06:42:34 +01:00
2023-01-02 07:51:41 +01:00
class SongMetadata
2023-01-02 10:31:55 +01:00
def initialize ( name : String , author : String , normalized_url : String , source : String , author_url : String , duration : Int32 | Nil , size : Int32 | Nil )
2023-01-01 06:42:34 +01:00
@name = name
@author = author
2023-01-02 07:51:41 +01:00
@normalized_url = normalized_url
@source = source
@author_url = author_url
@duration = duration
2023-01-02 10:31:55 +01:00
@size = size
2023-01-02 07:51:41 +01:00
end
def name
@name
end
def author
@author
end
def normalized_url
@normalized_url
end
def source
@source
end
def author_url
@author_url
end
def duration
@duration
2023-01-01 06:42:34 +01:00
end
2023-01-02 10:31:55 +01:00
def size
@size
end
2023-01-01 06:42:34 +01:00
end
def is_source_allowed ( source : String ) : Bool
config_get ( " songs.allow_all_sources " ) . as? ( Bool ) || config_get ( " songs.sources. #{ source } .allow " ) . as? ( Bool ) || false
end
2023-01-02 07:51:41 +01:00
def get_file_path ( song_id : Int32 )
2023-01-05 15:08:57 +01:00
DATA_FOLDER / " songs " / " #{ song_id } .mp3 "
2023-01-02 07:51:41 +01:00
end
# will raise errors
def fetch_song_metadata ( url : String ) : SongMetadata
2023-01-02 11:59:37 +01:00
LOG . info { " getting metadata for #{ url } " }
2023-01-01 06:42:34 +01:00
output = IO :: Memory . new
# todo: ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ LOOK OUT FOR SHELL INJECTION BULLSHIT!!!!!!!!!!!!!!!!!!
2023-01-07 09:54:13 +01:00
Process . run ( config_get ( " songs.sources.ytdlp_binary " , " yt-dlp " ) , [ " -J " , url ] , output : output )
2023-01-01 06:42:34 +01:00
output . close
metadata = JSON . parse ( output . to_s )
2023-01-02 07:51:41 +01:00
canonical_url = metadata [ " webpage_url " ] . as_s? || metadata [ " original_url " ] . as_s? || url
duration = metadata [ " duration " ]? && ( metadata [ " duration " ] . as_f? || metadata [ " duration " ] . as_i? )
return SongMetadata . new (
( metadata [ " fulltitle " ]? && metadata [ " fulltitle " ] . as_s? ) || ( metadata [ " title " ]? && metadata [ " title " ] . as_s? ) || url ,
( metadata [ " uploader " ]? && metadata [ " uploader " ] . as_s? ) || " " ,
canonical_url ,
metadata [ " extractor " ] . as_s ,
2023-01-02 10:31:55 +01:00
( metadata [ " uploader_url " ]? && metadata [ " uploader_url " ] . as_s? ) || canonical_url ,
duration ? duration . to_i : nil ,
metadata [ " filesize " ]? && metadata [ " filesize " ] . as_i?
2023-01-02 07:51:41 +01:00
)
end
2023-01-02 10:31:55 +01:00
def get_artist_id ( artist_name : String , artist_url : String , source : String ) : Int32
if source == " unknown "
return UNKNOWN_SONG_AUTHOR
end
2023-01-04 13:05:27 +01:00
if artist_name . blank?
2023-01-02 10:31:55 +01:00
return UNKNOWN_SONG_AUTHOR
end
begin
DATABASE . query_one ( " select id from song_authors where name = ? and url = ? and source = ? " , artist_name , artist_url , source , as : { Int32 } )
rescue
2023-01-02 14:32:31 +01:00
next_id = IDs . get_next_id ( " song_authors " )
2023-01-02 10:31:55 +01:00
DATABASE . exec ( " insert into song_authors (id, source, name, url) values (?, ?, ?, ?) " , next_id , source , artist_name , artist_url )
next_id . to_i
end
end
2023-01-02 07:51:41 +01:00
# name, author id, author name, size, download url
# returns nil if song should be disabled
# throws if something failed
2023-01-10 10:13:06 +01:00
# todo: why is size sometimes 0.00 despite the song being downloaded?
2023-01-02 07:51:41 +01:00
def fetch_song ( song_id : Int32 , get_download = false ) : Tuple ( String , Int32 , String , Int32 | Nil , String | Nil ) | Nil
2023-01-02 11:59:37 +01:00
LOG . debug { " fetching #{ song_id } " }
2023-01-02 07:51:41 +01:00
if ! config_get ( " songs.allow_custom_songs " ) . as? ( Bool )
return nil
end
# todo: this is kinda spaghetti
metadata = nil
author_id = nil
fetch_url = nil
song_exists = false
url = nil
begin
url , disabled = DATABASE . query_one ( " select url, disabled from songs where id = ? " , song_id , as : { String , Bool } )
if disabled
return nil
end
song_exists = true
rescue
if config_get ( " songs.preserve_newgrounds_ids " ) . as? ( Bool )
url = " https://www.newgrounds.com/audio/listen/ #{ song_id } "
else
raise " unknown song ID "
end
2023-01-01 06:42:34 +01:00
end
2023-01-02 07:51:41 +01:00
if DATABASE . scalar ( " select count(*) from song_data where id = ? " , song_id ) . as ( Int64 ) > 0
song_name , song_author_id , song_author_name , song_author_url , song_size , song_source , song_duration , download_url = DATABASE . query_one ( " select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ? " , song_id , as : { String , Int32 , String ?, String ?, Int32 ?, String , Int32 ?, String ?} )
2023-01-01 06:42:34 +01:00
2023-01-02 07:51:41 +01:00
fetch_url = download_url
author_id = song_author_id
2023-01-02 10:31:55 +01:00
metadata = SongMetadata . new ( song_name , song_author_name || " " , url . not_nil! , song_source , song_author_url || " " , song_duration , song_size )
2023-01-02 07:51:41 +01:00
else
2023-01-07 09:46:18 +01:00
if url == nil
# prevent disabling unoccupied IDs
return nil
end
2023-01-02 07:51:41 +01:00
begin
metadata = fetch_song_metadata ( url . not_nil! )
rescue err
2023-01-02 11:59:37 +01:00
LOG . warn { " ran into error fetching metadata: #{ err } ; disabling song " }
LOG . warn { err . inspect }
2023-01-02 07:51:41 +01:00
if song_exists
DATABASE . exec ( " update songs set disabled=1 where id = ? " , song_id )
else
DATABASE . exec ( " insert into songs (id, url, disabled) values (?, ?, 1) " , song_id , url )
end
2023-01-02 10:31:55 +01:00
return nil
2023-01-02 07:51:41 +01:00
else
if song_exists && url != metadata . normalized_url
DATABASE . exec ( " update songs set url = ? where id = ? " , metadata . normalized_url , song_id )
end
if DATABASE . scalar ( " select count(*) from songs join song_data on songs.id = song_data.id where songs.id != ? and url = ? " , song_id , metadata . normalized_url ) . as ( Int64 ) > 0
# just use that song's metadata instead
# todo: dedup this and the above similar block somehow?
song_name , song_author_id , song_author_name , song_author_url , song_size , song_source , song_duration , download_url = DATABASE . query_all ( " select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ? " , song_id , metadata . normalized_url , as : { String , Int32 , String ?, String ?, Int32 ?, String , Int32 ?, String ?} ) [ 0 ]
fetch_url = download_url
author_id = song_author_id
2023-01-02 10:31:55 +01:00
metadata = SongMetadata . new ( song_name , song_author_name || " " , url . not_nil! , song_source , song_author_url || " " , song_duration , song_size )
2023-01-02 07:51:41 +01:00
end
2023-01-01 06:42:34 +01:00
end
end
2023-01-02 11:59:37 +01:00
LOG . debug { metadata . inspect }
2023-01-02 07:51:41 +01:00
# do checks to make sure this is a valid song
max_duration = config_get ( " songs.sources.max_duration " ) . as? ( Int64 )
# todo
2023-01-02 10:31:55 +01:00
if ( fetch_url || ! get_download ) && metadata && author_id
2023-01-02 07:51:41 +01:00
# we're done! woo
if fetch_url && fetch_url . starts_with? ( " ./ " )
2023-01-07 09:54:13 +01:00
fetch_url = " #{ config_get ( " general.hostname " , " " ) } / #{ fetch_url [ 2 .. ] } "
2023-01-01 06:42:34 +01:00
end
2023-01-02 10:31:55 +01:00
return { metadata . name , author_id , metadata . author , metadata . size , fetch_url }
2023-01-02 07:51:41 +01:00
end
2023-01-01 06:42:34 +01:00
2023-01-02 07:51:41 +01:00
metadata = metadata . not_nil!
2023-01-02 10:31:55 +01:00
new_size = nil
2023-01-01 06:42:34 +01:00
2023-01-02 07:51:41 +01:00
if get_download
if config_get ( " songs.sources.allow_transcoding " )
if ! config_get ( " songs.sources.proxy_downloads " ) . as? ( Bool )
raise " can't download a song with transcoding but without proxying allowed "
end
2023-01-01 06:42:34 +01:00
2023-01-02 07:51:41 +01:00
# todo: check if song file exists
2023-01-01 06:42:34 +01:00
2023-01-02 07:51:41 +01:00
target_path = get_file_path ( song_id )
2023-01-01 06:42:34 +01:00
2023-01-07 09:54:13 +01:00
Process . run ( config_get ( " songs.sources.ytdlp_binary " , " yt-dlp " ) , [ " -f " , " ba " , " -x " , " --audio-format " , GD_AUDIO_FORMAT , " -o " , target_path . to_s , " --ffmpeg-location " , config_get ( " songs.sources.ffmpeg_binary " , " ffmpeg " ) , metadata . normalized_url ] , output : STDOUT , error : STDOUT )
2023-01-02 07:51:41 +01:00
2023-01-02 10:31:55 +01:00
new_size = File . size ( target_path ) . to_i
# todo: get duration
2023-01-02 07:51:41 +01:00
fetch_url = " ./ #{ song_id } .mp3 "
else
# todo
raise " fetching songs without transcoding and proxying downloads currently unimplemented "
end
end
if ! author_id
2023-01-02 10:31:55 +01:00
author_id = get_artist_id ( metadata . author , metadata . source , metadata . author_url )
end
if config_get ( " songs.sources.proxy_downloads " )
if DATABASE . scalar ( " select count(*) from song_data where id = ? " , song_id ) . as ( Int64 ) > 0
DATABASE . exec ( " update song_data set name = ?, author_id = ?, size = ? where id = ? " , metadata . name , author_id , new_size || metadata . size , song_id )
else
DATABASE . exec ( " insert into song_data (id, name, author_id, source, size, duration, proxy_url) values (?, ?, ?, ?, ?, ?, ?) " , song_id , metadata . name , author_id , metadata . source , metadata . size , metadata . duration , fetch_url )
end
else
2023-01-02 07:51:41 +01:00
# todo
end
if fetch_url && fetch_url . starts_with? ( " ./ " )
2023-01-03 18:23:34 +01:00
# todo: deduplicate this with similar block above?
fetch_url = " #{ config_get ( " general.hostname " ) . as? ( String ) || " " } / #{ fetch_url [ 2 .. ] } "
2023-01-01 06:42:34 +01:00
end
2023-01-02 10:31:55 +01:00
return { metadata . name , author_id , metadata . author , new_size || metadata . size , fetch_url }
2023-01-01 06:42:34 +01:00
end
2022-12-31 14:25:43 +01:00
end