#### Benben
#### Copyright (C) 2023 Remilia Scarlet <remilia@posteo.jp>
####
#### This program is free software: you can redistribute it and/or
#### modify it under the terms of the GNU Affero General Public
#### License as published by the Free Software Foundation, either
#### version 3 of the License, or (at your option) any later version.
####
#### This program is distributed in the hope that it will be useful,
#### but WITHOUT ANY WARRANTY; without even the implied warranty of
#### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
#### Affero General Public License for more details.
####
#### You should have received a copy of the GNU Affero General Public License
#### along with this program. If not, see <https://www.gnu.org/licenses/>.
require "./job"
module Benben::Rendering
class RenderingError < Exception
getter outFilename : Path?
def initialize(@outFilename, msg : String, cause : Exception)
super(msg, cause)
end
end
alias JobChannel = Channel(Bool|RenderingError)
alias ProgressChan = Channel(UInt32|Bool)
alias ProgressDoneChan = Channel(BigFloat)
alias ReceiverChan = Channel(Bool)
# The `Renderer` class acts as a manager for rendering VGM files in parallel.
class Renderer
@errors = [] of RenderingError
@baseOutputPath : String?
def initialize
end
# Renders all of the files that were specified on the command line (or
# indirectly via playlists).
def render : Nil
if Benben.fileHandler.size == 0
RemiLib.log.fatal("No VGMs found to render")
end
# Create various channels we'll use
receiverDoneChan : ReceiverChan = ReceiverChan.new(1)
pbarDoneChan : ProgressDoneChan = ProgressDoneChan.new(1)
# Keep track of how long this takes
start : Time = Time.local
# Loop, creating jobs for all the VGMs we have. Note that this will load
# all VGMs into RAM.
jobChan : JobChannel = JobChannel.new(Benben.args["jobs"].str.to_i32 + 3)
puts "Loading files into RAM..."
jobs : Array(Job) = buildJobs(jobChan)
RemiLib.log.fatal("Nothing to do :(") if jobs.empty?
# Start the receiver
startReceiver(jobChan, receiverDoneChan)
# Start the progress bar. We use our own progress bar, not the one in the
# Printer.
pbarChan : ProgressChan = startProgressBar(jobs, pbarDoneChan)
# Start the pool
format("Files to render: ~:d~%", jobs.size)
puts "Starting rendering jobs..."
pool : RemiLib::JobPool(Job) = RemiLib::JobPool(Job).new(Benben.args["jobs"].str.to_i32)
totalFileSize : UInt64 = 0u64
pool.sleepCheckInterval = 5.milliseconds
pool.run(jobs) do |job|
job.render(pbarChan)
totalFileSize += File.size(job.outFilename) if File.exists?(job.outFilename)
pbarChan.send(true)
Fiber.yield
end
# Tell the receiver fiber and progress bar fiber to stop.
jobChan.send(false)
pbarChan.send(false)
receiverDoneChan.receive
avgSamples : BigFloat = pbarDoneChan.receive
# Print any errors
printErrors(pool)
# Print how much time was taken
finish : Time = Time.local
delta : Time::Span = finish - start
format("Number of files rendered: ~:d~%", jobs.size - pool.errors.size - @errors.size)
format("Total size of rendered files: ~a~%", totalFileSize.prettySize)
format("Time taken: ~a~%", delta)
format("Average samples per second: ~:d~%", avgSamples.to_u64!)
end
# Creates an Array of Job instances based on the requested VGM files.
private def buildJobs(jobChan : JobChannel) : Array(Job)
# The array we'll store Jobs in
jobs : Array(Job) = [] of Job
# Used to check for duplicate output filenames.
outputPaths : Hash(Path, Bool) = {} of Path => Bool
# Get each VGM
while vgm = Benben.fileHandler.next
# Get the output path
outputPath : String = getOutputPath
outputPath = adjustOutputPathForVgm(outputPath, vgm)
# Check if the output path exists, and if it doesn't, attempt to create
# it.
unless Dir.exists?(outputPath)
begin
Dir.mkdir_p(outputPath)
rescue err : Exception
RemiLib.log.fatal("Could not create output path: #{err}")
next
end
end
# Construct the filename.
filename : Path = Path[Path[Benben.fileHandler.currentFilename].basename]
ext : String = filename.extension
filename = if ext.empty?
filename
else
Path[filename.to_s.gsub(ext, Benben.args["au"].called? ? ".au" : ".wav")]
end
filename = Path[outputPath, filename]
# Check for duplicate destination filenames.
if outputPaths.has_key?(filename)
RemiLib.log.error("Two files have the same output name, rendering only once: #{filename}")
next
else
outputPaths[filename] = true
end
# Check for an existing file.
if File.exists?(filename)
unless Benben.args["overwrite"].called?
RemiLib.log.error("File exists: #{filename}")
next
end
end
# All good, store the new job
jobs << Job.new(vgm.not_nil!, Path[Benben.fileHandler.currentFilename], filename, jobChan)
Fiber.yield
end
jobs
end
# Adjusts *outputPath* using information from the given VGM file.
# Substitutions only happen if the corresponding GD3 tag field is not an
# empty string after stripping.
#
# The following substitutions are made:
#
# * `%{system}`: Becomes the system name. The English name preferred.
# * `%{systemEn}`: Becomes the English system name.
# * `%{systemJp}`: Becomes the Japanese system name.
# * `%{game}`: Becomes the game name. The English name preferred.
# * `%{gameEn}`: Becomes the English game name.
# * `%{gameJp}`: Becomes the Japanese game name.
# * `%{artist}`: Becomes the artist name. The English name preferred.
# * `%{artistEn}`: Becomes the English artist name.
# * `%{artistJp}`: Becomes the Japanese artist name.
# * `%{releaseDate}`: Becomes the release date of the VGM.
# * `%{creator}`: Becomes the creator date of the VGM.
#
# If any of these template fields are present in the output path, but are
# empty in the GD3, then they are substituted with an "Unknown X", where X
# is appropriate for that field.
private def adjustOutputPathForVgm(outputPath : String, vgm : Yuno::VgmFile) : String
gd3 : Yuno::Gd3Tag = vgm.gd3Tag
###############
# Substitute system, preferring the English name
if outputPath.includes?("%{system}")
if !gd3.systemNameEn.strip.empty?
outputPath = outputPath.gsub("%{system}", gd3.systemNameEn.strip)
elsif !gd3.systemNameJp.strip.empty?
outputPath = outputPath.gsub("%{system}", gd3.systemNameJp.strip)
else
outputPath = outputPath.gsub("%{system}", "Unknown System")
end
end
# Substitute the system's Japanese name
if outputPath.includes?("%{systemJp}")
if !gd3.systemNameJp.strip.empty?
outputPath = outputPath.gsub("%{systemJp}", gd3.systemNameJp.strip)
else
outputPath = outputPath.gsub("%{systemJp}", "Unknown System")
end
end
# Substitute the system's English name
if outputPath.includes?("%{systemEn}")
if !gd3.systemNameEn.strip.empty?
outputPath = outputPath.gsub("%{systemEn}", gd3.systemNameEn.strip)
else
outputPath = outputPath.gsub("%{systemEn}", "Unknown System")
end
end
###############
# Substitute game title, preferring the English title
if outputPath.includes?("%{game}")
if !gd3.gameNameEn.strip.empty?
outputPath = outputPath.gsub("%{game}", gd3.gameNameEn.strip)
elsif !gd3.gameNameJp.strip.empty?
outputPath = outputPath.gsub("%{game}", gd3.gameNameJp.strip)
else
outputPath = outputPath.gsub("%{game}", "Unknown Game")
end
end
# Substitute the game's Japanese title
if outputPath.includes?("%{gameJp}")
if !gd3.gameNameJp.strip.empty?
outputPath = outputPath.gsub("%{gameJp}", gd3.gameNameJp.strip)
else
outputPath = outputPath.gsub("%{gameJp}", "Unknown Game")
end
end
# Substitute the game's English title
if outputPath.includes?("%{gameEn}")
if !gd3.gameNameEn.strip.empty?
outputPath = outputPath.gsub("%{gameEn}", gd3.gameNameEn.strip)
else
outputPath = outputPath.gsub("%{gameEn}", "Unknown Game")
end
end
###############
# Substitute artist name, preferring the English name.
if outputPath.includes?("%{artist}")
if !gd3.authorNameEn.strip.empty?
outputPath = outputPath.gsub("%{artist}", gd3.authorNameEn.strip)
elsif !gd3.authorNameJp.strip.empty?
outputPath = outputPath.gsub("%{artist}", gd3.authorNameJp.strip)
else
outputPath = outputPath.gsub("%{artist}", "Unknown Artist")
end
end
# Substitute the artist's Japanese name
if outputPath.includes?("%{artistJp}")
if !gd3.authorNameJp.strip.empty?
outputPath = outputPath.gsub("%{artistJp}", gd3.authorNameJp.strip)
else
outputPath = outputPath.gsub("%{artistJp}", "Unknown Artist")
end
end
# Substitute the artist's English name
if outputPath.includes?("%{artistEn}")
if !gd3.authorNameEn.strip.empty?
outputPath = outputPath.gsub("%{artistEn}", gd3.authorNameEn.strip)
else
outputPath = outputPath.gsub("%{artistEn}", "Unknown Artist")
end
end
###############
# Substitute release date
if outputPath.includes?("%{releaseDate}")
if !gd3.releaseDate.strip.empty?
outputPath = outputPath.gsub("%{releaseDate}", gd3.releaseDate.strip)
else
outputPath = outputPath.gsub("%{releaseDate}", "Unknown Release Date")
end
end
###############
# Substitute the VGM creator
if outputPath.includes?("%{creator}")
if !gd3.creator.strip.empty?
outputPath = outputPath.gsub("%{creator}", gd3.creator.strip)
else
outputPath = outputPath.gsub("%{creator}", "Unknown Creator")
end
end
outputPath
end
# Gets the base output directory where rendered files will be stored.
private def getOutputPath : String
if @baseOutputPath.nil?
# Memoize
@baseOutputPath = if Benben.args["outdir"].called?
Benben.args["outdir"].str
else
Path["."].expand.to_s
end
end
@baseOutputPath || raise "Failed to memoize baseOutputPath"
end
# Starts receiving messages from Job instances over `inChan`.
private def startReceiver(inChan : JobChannel, receiverDoneChan : ReceiverChan) : Nil
# Spawn a fiber.
spawn do
loop do
msg = inChan.receive
if msg.is_a?(Bool)
if msg
# Song was finished
GC.collect
else
# Time to exit
receiverDoneChan.send(true)
break
end
elsif msg.is_a?(RenderingError)
@errors << msg
else
RemiLib.log.error("Received weird message of type #{typeof(msg)} in receiver")
end
Fiber.yield
end # Loop do
end # Spawn
end
# Prints out any errors received by the JobPool.
private def printErrors(pool : RemiLib::JobPool) : Nil
unless @errors.empty? && pool.errors.empty?
RemiLib.log.warn("Errors encountered during rendering!")
@errors.each do |err|
RemiLib.log.error("#{err.outFilename}: #{err.message}")
end
pool.errors.each do |err|
{% if flag?(:benben_debug) %}
# This is rather ugly output-wise, but since it's only during
# debugging, it's not too concerning.
RemiLib.log.error("#{err} (#{err.backtrace})")
{% else %}
RemiLib.log.error("#{err}")
{% end %}
end
end
end
# Calculates the total number of samples, across all jobs, that we'll be
# rendering.
private def getTotalSamples(jobs : Array(Job)) : UInt64
jobs.sum(&.calcTotalSampleSize.to_u64!)
end
# Starts the progress bar, then returns a channel that can be used to
# communicate with the progress bar. When the progress bar exits, a message
# will be sent over `pbarDoneChan`.
private def startProgressBar(jobs : Array(Job), pbarDoneChan : ProgressDoneChan) : ProgressChan
totalSize : UInt64 = getTotalSamples(jobs)
totalFiles : Int32 = jobs.size
progressBar = RemiLib::Console::ProgressBar.new("Rendering", totalSize)
chan : ProgressChan = ProgressChan.new(Benben.args["jobs"].str.to_i32 + 3)
progressBar.noAutoRefresh = true
progressBar.postLabelWidth = 30u64
spawn do
numDone : UInt32 = 0
samplesDone : UInt64 = 0
lastUpdate : Time::Span = Time.monotonic
avgSamples : Int64 = 0
samplesLastPeriod : UInt64 = 0
allAvgSamples : BigFloat = BigFloat.new(0)
periods : UInt64 = 0
msg : UInt32|Bool = 0
loop do
msg = chan.receive
case msg
in UInt32
samplesDone += msg
samplesLastPeriod += msg
in Bool
if msg
numDone += 1
else
progressBar.label = "Rend: #{numDone}/#{totalFiles}"
progressBar.step = progressBar.max
progressBar.refresh
pbarDoneChan.send(allAvgSamples / (periods == 0 ? 1 : periods))
break
end
end
# Update the speed once per second
if (Time.monotonic - lastUpdate).total_milliseconds >= 500
avgSamples = (samplesLastPeriod /
((Time.monotonic - lastUpdate).total_nanoseconds / 500_000_000)).to_i64!
samplesLastPeriod = 0
allAvgSamples += avgSamples
periods += 1
lastUpdate = Time.monotonic
end
progressBar.label = "Rend: #{numDone}/#{totalFiles}"
progressBar.postLabel = formatStr("~:d Smp/s", avgSamples)
progressBar.step = samplesDone
progressBar.refresh
Fiber.yield
end
end
chan
end
end
end