polish up songs, refactor http server initialization
This commit is contained in:
parent
8b3a39da9d
commit
a7357c7f0d
|
@ -52,9 +52,6 @@ prevent_deletion_featured = true
|
||||||
# allow custom songs in general to be used,
|
# allow custom songs in general to be used,
|
||||||
# whether it be non-newgrounds or newgrounds ones
|
# whether it be non-newgrounds or newgrounds ones
|
||||||
allow_custom_songs = true
|
allow_custom_songs = true
|
||||||
# allow non-newgrounds custom songs to be used
|
|
||||||
# on the server
|
|
||||||
allow_nong_songs = true
|
|
||||||
# pushes all non-newgrounds songs above an arbitrary
|
# pushes all non-newgrounds songs above an arbitrary
|
||||||
# id to prevent collisions with newgrounds ids, meaning
|
# id to prevent collisions with newgrounds ids, meaning
|
||||||
# all song ids that work in vanilla GD will work
|
# all song ids that work in vanilla GD will work
|
||||||
|
@ -85,11 +82,17 @@ ffmpeg_binary = "/usr/bin/ffmpeg"
|
||||||
# required for allow_transcoding
|
# required for allow_transcoding
|
||||||
proxy_downloads = true
|
proxy_downloads = true
|
||||||
|
|
||||||
# expressed in seconds, doesn't affect NG
|
# expressed in seconds
|
||||||
max_duration = 600
|
max_duration = 600
|
||||||
|
|
||||||
|
# expressed in bytes
|
||||||
|
max_filesize = 10000000 # = 10MB
|
||||||
|
|
||||||
# see: https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md
|
# see: https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md
|
||||||
# not every source is supported, and most video sites will fail w/o transcoding enabled
|
# not every source is supported, and most video sites will fail w/o transcoding enabled
|
||||||
|
[songs.sources.newgrounds]
|
||||||
|
allow = true
|
||||||
|
|
||||||
[songs.sources.youtube]
|
[songs.sources.youtube]
|
||||||
allow = true
|
allow = true
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,7 @@ CREATE TABLE songs (
|
||||||
-- so this is a seperate table that's filled in for any given song once
|
-- so this is a seperate table that's filled in for any given song once
|
||||||
-- it's needed
|
-- it's needed
|
||||||
CREATE TABLE song_data (
|
CREATE TABLE song_data (
|
||||||
id SERIAL PRIMARY KEY references songs(id),
|
id SERIAL PRIMARY KEY references songs(id),
|
||||||
|
|
||||||
name TEXT NOT NULL,
|
name TEXT NOT NULL,
|
||||||
author_id INTEGER NOT NULL references song_authors(id),
|
author_id INTEGER NOT NULL references song_authors(id),
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
require "http/server"
|
require "http/server"
|
||||||
|
require "http/server/handler"
|
||||||
require "uri"
|
require "uri"
|
||||||
require "sqlite3"
|
require "sqlite3"
|
||||||
require "migrate"
|
require "migrate"
|
||||||
|
@ -41,38 +42,35 @@ module CrystalGauntlet
|
||||||
@@endpoints
|
@@endpoints
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.run()
|
class GDHandler
|
||||||
server = HTTP::Server.new do |context|
|
include HTTP::Handler
|
||||||
|
|
||||||
|
def call(context)
|
||||||
# expunge trailing slashes
|
# expunge trailing slashes
|
||||||
path = context.request.path.chomp("/")
|
path = context.request.path.chomp("/")
|
||||||
|
|
||||||
# todo: rethink life choices
|
path = path.sub(config_get("general.append_path").as(String | Nil) || "", "")
|
||||||
if path.ends_with?(".mp3")
|
|
||||||
# todo: BIG NONO
|
body = context.request.body
|
||||||
# todo: path traversal exploits SCARY
|
|
||||||
file = File.open("./data#{path}", "r")
|
if CrystalGauntlet.endpoints.has_key?(path) && body
|
||||||
context.response.content_type = "audio/mp3"
|
func = CrystalGauntlet.endpoints[path]
|
||||||
IO.copy(file, context.response)
|
value = func.call(body.gets_to_end)
|
||||||
file.close
|
context.response.content_type = "text/plain"
|
||||||
|
context.response.print value
|
||||||
else
|
else
|
||||||
path = path.sub(config_get("general.append_path").as(String | Nil) || "", "")
|
call_next(context)
|
||||||
|
|
||||||
body = context.request.body
|
|
||||||
|
|
||||||
if !body
|
|
||||||
puts "no body :("
|
|
||||||
elsif @@endpoints.has_key?(path)
|
|
||||||
func = @@endpoints[path]
|
|
||||||
value = func.call(body.gets_to_end)
|
|
||||||
context.response.content_type = "text/plain"
|
|
||||||
context.response.print value
|
|
||||||
puts "#{path} -> #{value}"
|
|
||||||
else
|
|
||||||
context.response.respond_with_status(404, "endpoint not found")
|
|
||||||
puts "#{path} -> 404"
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.run()
|
||||||
|
server = HTTP::Server.new([
|
||||||
|
HTTP::ErrorHandler.new,
|
||||||
|
HTTP::LogHandler.new,
|
||||||
|
CrystalGauntlet::GDHandler.new,
|
||||||
|
HTTP::StaticFileHandler.new("data/", directory_listing = true)
|
||||||
|
])
|
||||||
|
|
||||||
listen_on = URI.parse(ENV["LISTEN_ON"]? || "http://localhost:8080").normalize
|
listen_on = URI.parse(ENV["LISTEN_ON"]? || "http://localhost:8080").normalize
|
||||||
|
|
||||||
|
|
|
@ -25,13 +25,14 @@ module CrystalGauntlet::Songs
|
||||||
end
|
end
|
||||||
|
|
||||||
class SongMetadata
|
class SongMetadata
|
||||||
def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil)
|
def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil, size : Int32 | Nil)
|
||||||
@name = name
|
@name = name
|
||||||
@author = author
|
@author = author
|
||||||
@normalized_url = normalized_url
|
@normalized_url = normalized_url
|
||||||
@source = source
|
@source = source
|
||||||
@author_url = author_url
|
@author_url = author_url
|
||||||
@duration = duration
|
@duration = duration
|
||||||
|
@size = size
|
||||||
end
|
end
|
||||||
|
|
||||||
def name
|
def name
|
||||||
|
@ -52,6 +53,9 @@ module CrystalGauntlet::Songs
|
||||||
def duration
|
def duration
|
||||||
@duration
|
@duration
|
||||||
end
|
end
|
||||||
|
def size
|
||||||
|
@size
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def is_source_allowed(source : String) : Bool
|
def is_source_allowed(source : String) : Bool
|
||||||
|
@ -81,25 +85,41 @@ module CrystalGauntlet::Songs
|
||||||
(metadata["uploader"]? && metadata["uploader"].as_s?) || "",
|
(metadata["uploader"]? && metadata["uploader"].as_s?) || "",
|
||||||
canonical_url,
|
canonical_url,
|
||||||
metadata["extractor"].as_s,
|
metadata["extractor"].as_s,
|
||||||
(metadata.["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url,
|
(metadata["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url,
|
||||||
duration ? duration.to_i : nil
|
duration ? duration.to_i : nil,
|
||||||
|
metadata["filesize"]? && metadata["filesize"].as_i?
|
||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def get_artist_id(artist_name : String, artist_url : String, source : String) : Int32
|
||||||
|
if source == "unknown"
|
||||||
|
return UNKNOWN_SONG_AUTHOR
|
||||||
|
end
|
||||||
|
if artist_name == ""
|
||||||
|
return UNKNOWN_SONG_AUTHOR
|
||||||
|
end
|
||||||
|
|
||||||
|
begin
|
||||||
|
DATABASE.query_one("select id from song_authors where name = ? and url = ? and source = ?", artist_name, artist_url, source, as: {Int32})
|
||||||
|
rescue
|
||||||
|
next_id = (DATABASE.scalar("select max(id) from song_authors").as(Int64 | Nil) || 0) + 1
|
||||||
|
DATABASE.exec("insert into song_authors (id, source, name, url) values (?, ?, ?, ?)", next_id, source, artist_name, artist_url)
|
||||||
|
next_id.to_i
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
# name, author id, author name, size, download url
|
# name, author id, author name, size, download url
|
||||||
# returns nil if song should be disabled
|
# returns nil if song should be disabled
|
||||||
# throws if something failed
|
# throws if something failed
|
||||||
def fetch_song(song_id : Int32, get_download = false) : Tuple(String, Int32, String, Int32 | Nil, String | Nil) | Nil
|
def fetch_song(song_id : Int32, get_download = false) : Tuple(String, Int32, String, Int32 | Nil, String | Nil) | Nil
|
||||||
puts "fetching #{song_id}"
|
puts "fetching #{song_id}"
|
||||||
if !config_get("songs.allow_custom_songs").as?(Bool)
|
if !config_get("songs.allow_custom_songs").as?(Bool)
|
||||||
puts "custom songs not allowed"
|
|
||||||
return nil
|
return nil
|
||||||
end
|
end
|
||||||
|
|
||||||
# todo: this is kinda spaghetti
|
# todo: this is kinda spaghetti
|
||||||
metadata = nil
|
metadata = nil
|
||||||
author_id = nil
|
author_id = nil
|
||||||
size = nil
|
|
||||||
fetch_url = nil
|
fetch_url = nil
|
||||||
|
|
||||||
song_exists = false
|
song_exists = false
|
||||||
|
@ -125,9 +145,8 @@ module CrystalGauntlet::Songs
|
||||||
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_one("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ?", song_id, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})
|
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_one("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ?", song_id, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})
|
||||||
|
|
||||||
fetch_url = download_url
|
fetch_url = download_url
|
||||||
size = song_size
|
|
||||||
author_id = song_author_id
|
author_id = song_author_id
|
||||||
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration)
|
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size)
|
||||||
else
|
else
|
||||||
begin
|
begin
|
||||||
metadata = fetch_song_metadata(url.not_nil!)
|
metadata = fetch_song_metadata(url.not_nil!)
|
||||||
|
@ -139,6 +158,7 @@ module CrystalGauntlet::Songs
|
||||||
else
|
else
|
||||||
DATABASE.exec("insert into songs (id, url, disabled) values (?, ?, 1)", song_id, url)
|
DATABASE.exec("insert into songs (id, url, disabled) values (?, ?, 1)", song_id, url)
|
||||||
end
|
end
|
||||||
|
return nil
|
||||||
else
|
else
|
||||||
if song_exists && url != metadata.normalized_url
|
if song_exists && url != metadata.normalized_url
|
||||||
DATABASE.exec("update songs set url = ? where id = ?", metadata.normalized_url, song_id)
|
DATABASE.exec("update songs set url = ? where id = ?", metadata.normalized_url, song_id)
|
||||||
|
@ -151,31 +171,29 @@ module CrystalGauntlet::Songs
|
||||||
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_all("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ?", song_id, metadata.normalized_url, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})[0]
|
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_all("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ?", song_id, metadata.normalized_url, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})[0]
|
||||||
|
|
||||||
fetch_url = download_url
|
fetch_url = download_url
|
||||||
size = song_size
|
|
||||||
author_id = song_author_id
|
author_id = song_author_id
|
||||||
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration)
|
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
puts metadata.inspect
|
puts metadata.inspect
|
||||||
|
|
||||||
# todo: insert into song_data
|
|
||||||
|
|
||||||
# do checks to make sure this is a valid song
|
# do checks to make sure this is a valid song
|
||||||
max_duration = config_get("songs.sources.max_duration").as?(Int64)
|
max_duration = config_get("songs.sources.max_duration").as?(Int64)
|
||||||
# todo
|
# todo
|
||||||
|
|
||||||
if (fetch_url || !get_download) && metadata && size && author_id
|
if (fetch_url || !get_download) && metadata && author_id
|
||||||
# we're done! woo
|
# we're done! woo
|
||||||
if fetch_url && fetch_url.starts_with?("./")
|
if fetch_url && fetch_url.starts_with?("./")
|
||||||
# todo
|
# todo
|
||||||
fetch_url = "localhost:8080/#{fetch_url[2..]}"
|
fetch_url = "localhost:8080/#{fetch_url[2..]}"
|
||||||
end
|
end
|
||||||
return {metadata.name, author_id, metadata.author, size, fetch_url}
|
return {metadata.name, author_id, metadata.author, metadata.size, fetch_url}
|
||||||
end
|
end
|
||||||
|
|
||||||
metadata = metadata.not_nil!
|
metadata = metadata.not_nil!
|
||||||
|
new_size = nil
|
||||||
|
|
||||||
if get_download
|
if get_download
|
||||||
if config_get("songs.sources.allow_transcoding")
|
if config_get("songs.sources.allow_transcoding")
|
||||||
|
@ -189,20 +207,29 @@ module CrystalGauntlet::Songs
|
||||||
|
|
||||||
Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", metadata.normalized_url], output: STDOUT, error: STDOUT)
|
Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", metadata.normalized_url], output: STDOUT, error: STDOUT)
|
||||||
|
|
||||||
size = File.size(target_path).to_i
|
new_size = File.size(target_path).to_i
|
||||||
|
|
||||||
|
# todo: get duration
|
||||||
|
|
||||||
fetch_url = "./#{song_id}.mp3"
|
fetch_url = "./#{song_id}.mp3"
|
||||||
else
|
else
|
||||||
# todo
|
# todo
|
||||||
raise "fetching songs without transcoding and proxying downloads currently unimplemented"
|
raise "fetching songs without transcoding and proxying downloads currently unimplemented"
|
||||||
end
|
end
|
||||||
|
|
||||||
# todo: update song_data with size, duration and url
|
|
||||||
end
|
end
|
||||||
|
|
||||||
if !author_id
|
if !author_id
|
||||||
|
author_id = get_artist_id(metadata.author, metadata.source, metadata.author_url)
|
||||||
|
end
|
||||||
|
|
||||||
|
if config_get("songs.sources.proxy_downloads")
|
||||||
|
if DATABASE.scalar("select count(*) from song_data where id = ?", song_id).as(Int64) > 0
|
||||||
|
DATABASE.exec("update song_data set name = ?, author_id = ?, size = ? where id = ?", metadata.name, author_id, new_size || metadata.size, song_id)
|
||||||
|
else
|
||||||
|
DATABASE.exec("insert into song_data (id, name, author_id, source, size, duration, proxy_url) values (?, ?, ?, ?, ?, ?, ?)", song_id, metadata.name, author_id, metadata.source, metadata.size, metadata.duration, fetch_url)
|
||||||
|
end
|
||||||
|
else
|
||||||
# todo
|
# todo
|
||||||
author_id = 1
|
|
||||||
end
|
end
|
||||||
|
|
||||||
if fetch_url && fetch_url.starts_with?("./")
|
if fetch_url && fetch_url.starts_with?("./")
|
||||||
|
@ -210,6 +237,6 @@ module CrystalGauntlet::Songs
|
||||||
# todo also: deduplicate this with similar block above?
|
# todo also: deduplicate this with similar block above?
|
||||||
fetch_url = "localhost:8080/#{fetch_url[2..]}"
|
fetch_url = "localhost:8080/#{fetch_url[2..]}"
|
||||||
end
|
end
|
||||||
return {metadata.name, author_id, metadata.author, size, fetch_url}
|
return {metadata.name, author_id, metadata.author, new_size || metadata.size, fetch_url}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in New Issue