oat/crystal-gauntlet
oat
/
crystal-gauntlet
1
0
Fork 1
Code Issues 11 Pull Requests Packages Projects Releases Wiki Activity

polish up songs, refactor http server initialization

This commit is contained in:
Jill 2023-01-02 12:31:55 +03:00
parent 8b3a39da9d
commit a7357c7f0d
4 changed files with 77 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
config.example.toml
View File

@ -52,9 +52,6 @@ prevent_deletion_featured = true
# allow custom songs in general to be used,
# whether it be non-newgrounds or newgrounds ones
allow_custom_songs = true
# allow non-newgrounds custom songs to be used
# on the server
allow_nong_songs = true
# pushes all non-newgrounds songs above an arbitrary
# id to prevent collisions with newgrounds ids, meaning
# all song ids that work in vanilla GD will work
@ -85,11 +82,17 @@ ffmpeg_binary = "/usr/bin/ffmpeg"
# required for allow_transcoding
proxy_downloads = true
# expressed in seconds, doesn't affect NG
# expressed in seconds
max_duration = 600
# expressed in bytes
max_filesize = 10000000 # = 10MB
# see: https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md
# not every source is supported, and most video sites will fail w/o transcoding enabled
[songs.sources.newgrounds]
allow = true
[songs.sources.youtube]
allow = true

2
db/migrations/6_songs.sql
View File

@ -10,7 +10,7 @@ CREATE TABLE songs (
-- so this is a seperate table that's filled in for any given song once
-- it's needed
CREATE TABLE song_data (
id SERIAL PRIMARY KEY references songs(id),
id SERIAL PRIMARY KEY references songs(id),
name TEXT NOT NULL,
author_id INTEGER NOT NULL references song_authors(id),

50
src/crystal-gauntlet.cr
View File

@ -1,4 +1,5 @@
require "http/server"
require "http/server/handler"
require "uri"
require "sqlite3"
require "migrate"
@ -41,38 +42,35 @@ module CrystalGauntlet
@@endpoints
end
def self.run()
server = HTTP::Server.new do |context|
class GDHandler
include HTTP::Handler
def call(context)
# expunge trailing slashes
path = context.request.path.chomp("/")
# todo: rethink life choices
if path.ends_with?(".mp3")
# todo: BIG NONO
# todo: path traversal exploits SCARY
file = File.open("./data#{path}", "r")
context.response.content_type = "audio/mp3"
IO.copy(file, context.response)
file.close
path = path.sub(config_get("general.append_path").as(String | Nil) || "", "")
body = context.request.body
if CrystalGauntlet.endpoints.has_key?(path) && body
func = CrystalGauntlet.endpoints[path]
value = func.call(body.gets_to_end)
context.response.content_type = "text/plain"
context.response.print value
else
path = path.sub(config_get("general.append_path").as(String | Nil) || "", "")
body = context.request.body
if !body
puts "no body :("
elsif @@endpoints.has_key?(path)
func = @@endpoints[path]
value = func.call(body.gets_to_end)
context.response.content_type = "text/plain"
context.response.print value
puts "#{path} -> #{value}"
else
context.response.respond_with_status(404, "endpoint not found")
puts "#{path} -> 404"
end
call_next(context)
end
end
end
def self.run()
server = HTTP::Server.new([
HTTP::ErrorHandler.new,
HTTP::LogHandler.new,
CrystalGauntlet::GDHandler.new,
HTTP::StaticFileHandler.new("data/", directory_listing = true)
])
listen_on = URI.parse(ENV["LISTEN_ON"]? || "http://localhost:8080").normalize

63
src/lib/songs.cr
View File

@ -25,13 +25,14 @@ module CrystalGauntlet::Songs
end
class SongMetadata
def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil)
def initialize(name : String, author : String, normalized_url : String, source : String, author_url : String, duration : Int32 | Nil, size : Int32 | Nil)
@name = name
@author = author
@normalized_url = normalized_url
@source = source
@author_url = author_url
@duration = duration
@size = size
end
def name
@ -52,6 +53,9 @@ module CrystalGauntlet::Songs
def duration
@duration
end
def size
@size
end
end
def is_source_allowed(source : String) : Bool
@ -81,25 +85,41 @@ module CrystalGauntlet::Songs
(metadata["uploader"]? && metadata["uploader"].as_s?) || "",
canonical_url,
metadata["extractor"].as_s,
(metadata.["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url,
duration ? duration.to_i : nil
(metadata["uploader_url"]? && metadata["uploader_url"].as_s?) || canonical_url,
duration ? duration.to_i : nil,
metadata["filesize"]? && metadata["filesize"].as_i?
)
end
def get_artist_id(artist_name : String, artist_url : String, source : String) : Int32
if source == "unknown"
return UNKNOWN_SONG_AUTHOR
end
if artist_name == ""
return UNKNOWN_SONG_AUTHOR
end
begin
DATABASE.query_one("select id from song_authors where name = ? and url = ? and source = ?", artist_name, artist_url, source, as: {Int32})
rescue
next_id = (DATABASE.scalar("select max(id) from song_authors").as(Int64 | Nil) || 0) + 1
DATABASE.exec("insert into song_authors (id, source, name, url) values (?, ?, ?, ?)", next_id, source, artist_name, artist_url)
next_id.to_i
end
end
# name, author id, author name, size, download url
# returns nil if song should be disabled
# throws if something failed
def fetch_song(song_id : Int32, get_download = false) : Tuple(String, Int32, String, Int32 | Nil, String | Nil) | Nil
puts "fetching #{song_id}"
if !config_get("songs.allow_custom_songs").as?(Bool)
puts "custom songs not allowed"
return nil
end
# todo: this is kinda spaghetti
metadata = nil
author_id = nil
size = nil
fetch_url = nil
song_exists = false
@ -125,9 +145,8 @@ module CrystalGauntlet::Songs
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_one("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join song_authors on song_authors.id = song_data.author_id where song_data.id = ?", song_id, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})
fetch_url = download_url
size = song_size
author_id = song_author_id
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration)
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size)
else
begin
metadata = fetch_song_metadata(url.not_nil!)
@ -139,6 +158,7 @@ module CrystalGauntlet::Songs
else
DATABASE.exec("insert into songs (id, url, disabled) values (?, ?, 1)", song_id, url)
end
return nil
else
if song_exists && url != metadata.normalized_url
DATABASE.exec("update songs set url = ? where id = ?", metadata.normalized_url, song_id)
@ -151,31 +171,29 @@ module CrystalGauntlet::Songs
song_name, song_author_id, song_author_name, song_author_url, song_size, song_source, song_duration, download_url = DATABASE.query_all("select song_data.name, author_id, song_authors.name, song_authors.url, size, song_data.source, duration, proxy_url from song_data left join songs on song_data.id = songs.id left join song_authors on song_authors.id = song_data.author_id where song_data.id != ? and songs.url = ?", song_id, metadata.normalized_url, as: {String, Int32, String?, String?, Int32?, String, Int32?, String?})[0]
fetch_url = download_url
size = song_size
author_id = song_author_id
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration)
metadata = SongMetadata.new(song_name, song_author_name || "", url.not_nil!, song_source, song_author_url || "", song_duration, song_size)
end
end
end
puts metadata.inspect
# todo: insert into song_data
# do checks to make sure this is a valid song
max_duration = config_get("songs.sources.max_duration").as?(Int64)
# todo
if (fetch_url || !get_download) && metadata && size && author_id
if (fetch_url || !get_download) && metadata && author_id
# we're done! woo
if fetch_url && fetch_url.starts_with?("./")
# todo
fetch_url = "localhost:8080/#{fetch_url[2..]}"
end
return {metadata.name, author_id, metadata.author, size, fetch_url}
return {metadata.name, author_id, metadata.author, metadata.size, fetch_url}
end
metadata = metadata.not_nil!
new_size = nil
if get_download
if config_get("songs.sources.allow_transcoding")
@ -189,20 +207,29 @@ module CrystalGauntlet::Songs
Process.run(config_get("songs.sources.ytdlp_binary").as?(String) || "yt-dlp", ["-f", "ba", "-x", "--audio-format", GD_AUDIO_FORMAT, "-o", target_path.to_s, "--ffmpeg-location", config_get("songs.sources.ffmpeg_binary").as?(String) || "ffmpeg", metadata.normalized_url], output: STDOUT, error: STDOUT)
size = File.size(target_path).to_i
new_size = File.size(target_path).to_i
# todo: get duration
fetch_url = "./#{song_id}.mp3"
else
# todo
raise "fetching songs without transcoding and proxying downloads currently unimplemented"
end
# todo: update song_data with size, duration and url
end
if !author_id
author_id = get_artist_id(metadata.author, metadata.source, metadata.author_url)
end
if config_get("songs.sources.proxy_downloads")
if DATABASE.scalar("select count(*) from song_data where id = ?", song_id).as(Int64) > 0
DATABASE.exec("update song_data set name = ?, author_id = ?, size = ? where id = ?", metadata.name, author_id, new_size || metadata.size, song_id)
else
DATABASE.exec("insert into song_data (id, name, author_id, source, size, duration, proxy_url) values (?, ?, ?, ?, ?, ?, ?)", song_id, metadata.name, author_id, metadata.source, metadata.size, metadata.duration, fetch_url)
end
else
# todo
author_id = 1
end
if fetch_url && fetch_url.starts_with?("./")
@ -210,6 +237,6 @@ module CrystalGauntlet::Songs
# todo also: deduplicate this with similar block above?
fetch_url = "localhost:8080/#{fetch_url[2..]}"
end
return {metadata.name, author_id, metadata.author, size, fetch_url}
return {metadata.name, author_id, metadata.author, new_size || metadata.size, fetch_url}
end
end