diff --git a/scripts/loaders/SoundLoader.gd b/scripts/loaders/SoundLoader.gd index 3a7cf30..dae5a1f 100644 --- a/scripts/loaders/SoundLoader.gd +++ b/scripts/loaders/SoundLoader.gd @@ -166,24 +166,15 @@ func load_sfx_samples_data(snes_data: Dictionary, buffer: StreamPeerBuffer): # Called when the node enters the scene tree for the first time. func load_samples(snes_data: Dictionary, buffer: StreamPeerBuffer): load_sfx_samples_data(snes_data, buffer) - # var largest_sample_idx := -1 - # var largest_sample_sample_count := 0 - # var total_frames := 0 # For some reason, this is a bit slow currently under certain editor conditions. Might optimize later. for i in INST_NUM: var samp := get_inst_sample_data(snes_data, buffer, i) instrument_samples.append(samp) - # total_frames += samp.loop_end - # if largest_sample_sample_count < samp.loop_end: - # largest_sample_sample_count = samp.loop_end - # largest_sample_idx = i # Workaround for Godot 3.x quirk where looping samples are interpolated as if they go to nothing instead of looping instrument_samples_HACK_EXTENDED_LOOPS.append(HACK_EXTEND_LOOP_SAMPLE(samp)) # print('Instrument %02X has mix_rate %d Hz and %d samples'%[i, samp.mix_rate, len(samp.data)/2]) emit_signal('audio_inst_sample_loaded', i) - # samp.save_to_wav('output/instrument%02d(%dHz)(loop from %d).wav' % [i, samp.mix_rate, samp.loop_begin]) - # print('Largest sample is instrument %d with length %d and mix_rate %d'%[largest_sample_idx, largest_sample_sample_count, instrument_samples[largest_sample_idx].mix_rate]) - # print('Total frames: %d'%total_frames) + samp.save_to_wav('output/instrument%02d(%dHz)(loop from %d).wav' % [i, samp.mix_rate, samp.loop_begin]) # We start the texture with a bunch of same-size headers @@ -201,12 +192,8 @@ func samples_to_texture(): var header_length := num_samples * 6 # Create header and unwrapped payload separately first - var header_data := PoolByteArray() var header_buffer := StreamPeerBuffer.new() - header_buffer.data_array = header_data - var payload_data := PoolByteArray() var payload_buffer := StreamPeerBuffer.new() - payload_buffer.data_array = payload_data for sample in instrument_samples + sfx_samples: var loop_end: int = sample.loop_end @@ -215,7 +202,7 @@ func samples_to_texture(): if nonlooping: loop_begin = loop_end loop_end += 1 - header_buffer.put_u16(header_length + (len(payload_data)/2) + FILTER_PAD) # sample_start + header_buffer.put_u16(header_length + (payload_buffer.get_position()/2) + FILTER_PAD) # sample_start header_buffer.put_u16(sample.loop_end + FILTER_PAD) # sample_length header_buffer.put_u16(sample.loop_begin + FILTER_PAD) # sample_loop_begin header_buffer.put_u16(sample.mix_rate) # sample_mixrate @@ -239,7 +226,7 @@ func samples_to_texture(): payload_buffer.seek(pos) payload_buffer.put_16(frame) # Combine the unwrapped arrays - var data := header_data + payload_data + var data := header_buffer.data_array + payload_buffer.data_array # Now calculate wrapping and rowwise padding for the combined array for row in TEX_WIDTH: var row_end: int = (row + 1) * TEX_WIDTH * 2 # Remember: 8bit array, 16bit values diff --git a/shaders/audio_renderer.gdshader b/shaders/audio_renderer.gdshader index 01561f3..ba349d0 100644 --- a/shaders/audio_renderer.gdshader +++ b/shaders/audio_renderer.gdshader @@ -17,8 +17,6 @@ const float x10000 = float(0x10000); // 65536.0 const vec2 INT16_DOT_BE = vec2(xFF00, x00FF); const vec2 INT16_DOT_LE = vec2(x00FF, xFF00); -uniform sampler2D tex : hint_normal; - float unpack_uint16(vec2 uint16) { // Convert packed 2byte integer, sampled as two [0.0, 1.0] range floats, to the original int value [0, 65535] in float32 return dot(uint16, INT16_DOT_LE); @@ -45,7 +43,7 @@ vec2 pack_float_to_int16(float value) { return vec2(LSB, MSB); } -vec4 test_writeback(vec2 uv) { +vec4 test_writeback(sampler2D tex, vec2 uv) { // Test importing and exporting the samples, // and exporting a value derived from the UV vec4 output; @@ -131,7 +129,8 @@ const int NUM_CHANNEL_NOTE_PROBES = 11; // log2(MAX_CHANNEL_NOTE_EVENTS) uniform sampler2D midi_events; uniform vec2 midi_events_size = vec2(2048.0, 16.0); // SDR rendering only gives us [0.0, 1.0] from the sampler2D so we need to rescale it. -uniform float t_scale = 524.0; // Change this if we need longer than 8min44sec. +uniform float sdr_scale = 128.0; //1024.0; +// uniform float t_scale = 524.0; // Change this if we need longer than 8min44sec. // ^ Other things will also need changing, since 4096x4096 = 8MSamples is barely over 524 seconds at 32kHz. vec4 get_midi_texel(float x, float y) { return texture(midi_events, vec2(x, y)/midi_events_size).xyzw; @@ -151,31 +150,38 @@ vec4 render_song(float sample_progress) { vec2 downmixed_stereo = vec2(0.0); // Binary search the channels - for (int channel = 0; channel < NUM_CHANNELS; channel++) { + for (int channel = 0; channel < 1; channel++) { + // for (int channel = 0; channel < NUM_CHANNELS; channel++) { float row = float(channel * 2); float event_idx = 0.0; for (int i = 0; i < NUM_CHANNEL_NOTE_PROBES; i++) { float step_size = exp2(float(NUM_CHANNEL_NOTE_PROBES - i - 1)); - vec4 note_event = get_midi_texel(event_idx + step_size, row); + vec4 note_event = get_midi_texel(event_idx + step_size, row) * sdr_scale; float t_start = note_event.x; event_idx += (t >= t_start) ? step_size : 0.0; } - vec4 note_event = get_midi_texel(event_idx, row); - vec4 note_event_supplement = get_midi_texel(event_idx, row+1.0); - float t_start = note_event.x * t_scale; - float t_end = note_event.y * t_scale; - vec2 instrument_and_pitch = unpack_float(note_event.z); - float instrument_idx = instrument_and_pitch.x * 1023.0; - float pitch_idx = instrument_and_pitch.y * 1023.0; // TODO: Maybe rescale this for fine tuning? Don't use it raw because 2^(127-71) is MASSIVE, keep the power-of-2 calcs in shader. - vec2 velocity_and_pan = unpack_float(note_event_supplement.w); // Can leave these as [0.0, 1.0] and then mix appropriately - float velocity = velocity_and_pan.x; - float pan = velocity_and_pan.y; - vec2 attack_and_decay = unpack_float(note_event_supplement.x); - vec2 sustain_and_release = unpack_float(note_event_supplement.y); + vec4 note_event = get_midi_texel(event_idx, row) * sdr_scale; // scaled to [0.0, 1024.0] + vec4 note_event_supplement = get_midi_texel(event_idx, row+1.0); // left as [0.0, 1.0] + float t_start = note_event.x; + float t_end = note_event.y; + // ====================At some point I'll look back into packing floats==================== + // vec2 instrument_and_pitch = unpack_float(note_event.z); + // float instrument_idx = instrument_and_pitch.x * 1023.0; + // float pitch_idx = instrument_and_pitch.y * 1023.0; // TODO: Maybe rescale this for fine tuning? Don't use it raw because 2^(127-71) is MASSIVE, keep the power-of-2 calcs in shader. + // vec2 velocity_and_pan = unpack_float(note_event.w); // Can leave these as [0.0, 1.0] and then mix appropriately + // float velocity = velocity_and_pan.x; + // float pan = velocity_and_pan.y; + // vec2 attack_and_decay = unpack_float(note_event_supplement.x); + // vec2 sustain_and_release = unpack_float(note_event_supplement.y); // TBD = note_event_supplement.zw; - tremolo/vibrato/noise/pan_lfo/pitchbend/echo remain + // ====================At some point I'll look back into packing floats==================== + float instrument_idx = note_event.z; + float pitch_idx = note_event.w; + float velocity = note_event_supplement.x; + float pan = note_event_supplement.y; // For now, just branch this - if (t_end > t) { + if (t < t_end) { float samp = get_instrument_sample(instrument_idx, get_pitch_scale(pitch_idx), t-t_start, t_end-t_start); samp *= velocity; // TODO: do some ADSR here? @@ -183,15 +189,17 @@ vec4 render_song(float sample_progress) { } } // Convert the stereo float audio to S16LE - return vec4(pack_float_to_int16(downmixed_stereo.x), pack_float_to_int16(downmixed_stereo.y)); + // return vec4(pack_float_to_int16(downmixed_stereo.x), pack_float_to_int16(downmixed_stereo.y)); + return vec4(pack_float_to_int16(downmixed_stereo.x), pack_float_to_int16(mod(t, 2.0) - 1.0)); + // return vec4(pack_float_to_int16((t/10.0) - 1.0), pack_float_to_int16(mod(t, 2.0) - 1.0)); } void fragment() { // GLES2 vec2 uv = vec2(UV.x, 1.0-UV.y); - uv = (trunc(uv*UV_QUANTIZE)+0.5)/UV_QUANTIZE; - // COLOR.xyzw = test_writeback(uv); - COLOR.xyzw = render_song(dot(uv, vec2(1.0, midi_events_size.x))); + // uv = (trunc(uv*UV_QUANTIZE)+0.5)/UV_QUANTIZE; + // COLOR.xyzw = test_writeback(TEXTURE, uv); + COLOR.xyzw = render_song(dot(trunc(uv*TEX_SIZE), vec2(1.0, TEX_SIZE))); } // const int MAX_TEMPO_EVENTS = 256; diff --git a/test/audio_renderer.gd b/test/audio_renderer.gd index 8aadbe1..603929a 100644 --- a/test/audio_renderer.gd +++ b/test/audio_renderer.gd @@ -1,7 +1,11 @@ extends Control +const INPUT_TEX_WIDTH := 2048 +const INPUT_FORMAT := Image.FORMAT_RGBAF # Image.FORMAT_LA8 +const INPUT_BYTES_PER_TEXEL := 16 # 2 +const OUTPUT_WIDTH := 4096 var viewport: Viewport -var render_queue: Array # of PoolByteArrays +var render_queue: Array # of Images var result_queue: Array # of PoolByteArrays var current_image: Image var current_tex: ImageTexture # Needed to prevent GC before draw @@ -17,6 +21,18 @@ func _ready() -> void: self.current_image = Image.new() self.current_tex = ImageTexture.new() +func push_image(img: Image) -> void: + self.render_queue.append(img) + +func push_bytes(data: PoolByteArray) -> void: + # print(data.subarray(0, 15)) + var rows = int(pow(2, ceil(log((len(data)/INPUT_BYTES_PER_TEXEL) / INPUT_TEX_WIDTH)/log(2)))) + var target_length = rows * INPUT_BYTES_PER_TEXEL * INPUT_FORMAT + while len(data) < target_length: # This is inefficient, but this function should be called with pre-padded data anyway + data.append(0) + self.current_image.create_from_data(INPUT_TEX_WIDTH, rows, false, INPUT_FORMAT, data) + self.render_queue.append(self.current_image) + func _process(_delta) -> void: update() @@ -35,12 +51,13 @@ func _draw() -> void: return # Draw the next ImageTexture - var data: PoolByteArray = self.render_queue.pop_front() - print(data.subarray(0, 15)) - self.current_image.create_from_data(4096, 4096, false, Image.FORMAT_LA8, data) - self.current_tex.create_from_image(self.current_image, Texture.FLAG_FILTER) - self.material.set_shader_param('tex', self.current_tex) - draw_texture(self.current_tex, Vector2.ZERO) + self.current_image = self.render_queue.pop_front() + self.current_tex.create_from_image(self.current_image, 0) + self.material.set_shader_param('midi_events', self.current_tex) + self.material.set_shader_param('midi_events_size', self.current_tex.get_size()) + # draw_texture(self.current_tex, Vector2.ZERO) + draw_texture(self.viewport.get_texture(), Vector2.ZERO) + # draw_rect(Rect2(0, 0, OUTPUT_WIDTH, OUTPUT_WIDTH), Color.white) self.waiting_for_viewport = true # Grab the result next draw func get_result() -> void: @@ -48,13 +65,13 @@ func get_result() -> void: var result_image := result_texture.get_data() var result_bytes := result_image.get_data() - # Debugging: compare a sequence of all the possible 16bit integers - print_debug('result_image format is %d and has size'%result_image.get_format(), result_image.get_size(), result_bytes.subarray(0, 11)) - test_readback(result_bytes) - self.result_queue.append(result_bytes) self.waiting_for_viewport = false + # # Debugging: compare a sequence of all the possible 16bit integers + # print_debug('result_image format is %d and has size'%result_image.get_format(), result_image.get_size(), result_bytes.subarray(0, 11)) + # test_readback(result_bytes) + func test_readback(result_bytes: PoolByteArray): # Debugging: compare a sequence of all the possible 16bit integers var buff := StreamPeerBuffer.new() diff --git a/test/audio_system.gd b/test/audio_system.gd index 3841bbf..9c0d289 100644 --- a/test/audio_system.gd +++ b/test/audio_system.gd @@ -125,22 +125,71 @@ func _ready() -> void: for i in len(RomLoader.snes_data.bgm_song_pointers): var pointer = RomLoader.snes_data.bgm_song_pointers[i] # print('BGM 0x%02X (%02d) at 0x%06X' % [i, i, pointer]) + self.test_rendering() + +onready var audio_renderer := $'%audio_renderer' +func test_rendering() -> void: + SoundLoader.samples_to_texture() + audio_renderer.material.set_shader_param('instrument_samples', SoundLoader.samples_tex) + audio_renderer.material.set_shader_param('instrument_samples_size', SoundLoader.samples_tex.get_size()) + + var midi_events_bytes := StreamPeerBuffer.new() + var midi_events_bytes2 := StreamPeerBuffer.new() + var divisor = 128.0 #1024.0 # See sdr_scale in audio_renderer.gdshader + for i in 2048: + var t = i * 10.0 + midi_events_bytes.put_float(t/divisor) # t_start + midi_events_bytes.put_float((t+3.0)/divisor) # t_end + # Try repacking these later + midi_events_bytes.put_float((i%35)/divisor) # instrument + midi_events_bytes.put_float(71/divisor) # pitch_idx + # midi_events_bytes.put_float((35 + (i%40))/divisor) # pitch_idx + midi_events_bytes2.put_float(1.0) # velocity + midi_events_bytes2.put_float((i%101)/100.0) # pan + midi_events_bytes2.put_float(0.0) # TBD + midi_events_bytes2.put_float(0.0) # TBD + # midi_events_bytes.put_float(((i%35) + 71/1024.0)/1023.0) # instrument_and_pitch + # midi_events_bytes.put_float((1.0 + (i*4)/1024.0)/1023.0) # velocity_and_pan + # midi_events_bytes2.put_float(0.0) # attack_and_decay + # midi_events_bytes2.put_float(0.0) # sustain_and_relase + # midi_events_bytes2.put_float(0.0) # TBD + # midi_events_bytes2.put_float(0.0) # TBD + # for i in 2048-256: + # midi_events_bytes.put_float(0.0) + # midi_events_bytes.put_float(0.0) + # midi_events_bytes.put_float(0.0) + # midi_events_bytes.put_float(0.0) + # midi_events_bytes2.put_float(0.0) + # midi_events_bytes2.put_float(0.0) + # midi_events_bytes2.put_float(0.0) + # midi_events_bytes2.put_float(0.0) + var channel_data = midi_events_bytes.data_array + midi_events_bytes2.data_array + audio_renderer.push_bytes(channel_data) # + channel_data + channel_data + channel_data + channel_data + channel_data + channel_data + channel_data) # var test_payload := PoolByteArray() # test_payload.resize(4096*4096*2) # # for i in 5: # # test_payload.fill(i*2+10) -# # $'%audio_renderer'.render_queue.append(test_payload) +# # audio_renderer.render_queue.append(test_payload) # test_payload.fill(0) # for i in 65536: # test_payload.set(i*2, i%256) # test_payload.set(i*2+1, i/256) -# $'%audio_renderer'.render_queue.append(test_payload) -# # $'%audio_renderer'.render_queue.append(test_payload) +# audio_renderer.render_queue.append(test_payload) +# # audio_renderer.render_queue.append(test_payload) -# func _process(_delta): -# update() +func _process(_delta): + update() -# func _draw() -> void: -# if $'%audio_renderer'.waiting_for_viewport: -# $'%audio_renderer'.get_result() +func _draw() -> void: + if audio_renderer.waiting_for_viewport: + audio_renderer.get_result() + var result = audio_renderer.result_queue[0] + var rendered_audio := AudioStreamSample.new() + rendered_audio.data = result + rendered_audio.stereo = true + rendered_audio.mix_rate = 32000 + rendered_audio.format = AudioStreamSample.FORMAT_16_BITS + var error = rendered_audio.save_to_wav('output/rendered_audio.wav') + print(error) + pass diff --git a/test/audio_system.tscn b/test/audio_system.tscn index c92fcd3..2aef8f7 100644 --- a/test/audio_system.tscn +++ b/test/audio_system.tscn @@ -7,6 +7,13 @@ [sub_resource type="ShaderMaterial" id=2] shader = ExtResource( 4 ) +shader_param/instrument_samples_size = Vector2( 2048, 128 ) +shader_param/instrument_row_padding = 3.0 +shader_param/instrument_row_payload = 2042.0 +shader_param/reference_note = 71.0 +shader_param/output_mixrate = 32000.0 +shader_param/midi_events_size = Vector2( 2048, 16 ) +shader_param/t_scale = 524.0 [node name="audio_system" type="Node2D"] script = ExtResource( 1 )