birdulon/ChocolateBird
birdulon
/
ChocolateBird
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[WIP] Audio shader almost working

This commit is contained in:
Luke Hubmayer-Werner 2024-07-12 01:56:05 +09:30
parent e077c4e036
commit fbd3dd52c7
4 changed files with 27 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
scripts/loaders/SoundLoader.gd
View File

@ -189,7 +189,7 @@ const TEX_WIDTH := 2048
const FILTER_PAD := 3 const FILTER_PAD := 3
func samples_to_texture(): func samples_to_texture():
var num_samples := INST_NUM + SFX_NUM var num_samples := INST_NUM + SFX_NUM
var header_length := num_samples * 6 var header_length := num_samples * 8
# Create header and unwrapped payload separately first # Create header and unwrapped payload separately first
var header_buffer := StreamPeerBuffer.new() var header_buffer := StreamPeerBuffer.new()
@ -201,15 +201,16 @@ func samples_to_texture():
var nonlooping: bool = loop_begin >= loop_end var nonlooping: bool = loop_begin >= loop_end
if nonlooping: if nonlooping:
loop_begin = loop_end loop_begin = loop_end
loop_end += 1 loop_end += 3
header_buffer.put_u16(header_length + (payload_buffer.get_position()/2) + FILTER_PAD) # sample_start header_buffer.put_32(header_length + (payload_buffer.get_position()/2) + FILTER_PAD) # sample_start
header_buffer.put_u16(sample.loop_end + FILTER_PAD) # sample_length header_buffer.put_u16(loop_end + FILTER_PAD) # sample_length
header_buffer.put_u16(sample.loop_begin + FILTER_PAD) # sample_loop_begin header_buffer.put_u16(loop_begin + FILTER_PAD) # sample_loop_begin
header_buffer.put_u16(sample.mix_rate) # sample_mixrate header_buffer.put_u16(sample.mix_rate) # sample_mixrate
header_buffer.put_u8(0) # TODO: attack header_buffer.put_u8(0) # TODO: attack
header_buffer.put_u8(0) # TODO: decay header_buffer.put_u8(0) # TODO: decay
header_buffer.put_u8(0) # TODO: sustain header_buffer.put_u8(0) # TODO: sustain
header_buffer.put_u8(0) # TODO: release header_buffer.put_u8(0) # TODO: release
header_buffer.put_u16(0) # TODO: unk
for i in FILTER_PAD: # Prepend 3 frames of silence for i in FILTER_PAD: # Prepend 3 frames of silence
payload_buffer.put_16(0) payload_buffer.put_16(0)
payload_buffer.put_data(sample.data) # Copy entire S16LE audio data payload_buffer.put_data(sample.data) # Copy entire S16LE audio data
@ -251,7 +252,7 @@ func samples_to_texture():
var samples_img = Image.new() var samples_img = Image.new()
samples_img.create_from_data(TEX_WIDTH, rows, false, Image.FORMAT_LA8, data) samples_img.create_from_data(TEX_WIDTH, rows, false, Image.FORMAT_LA8, data)
self.samples_tex = ImageTexture.new() self.samples_tex = ImageTexture.new()
self.samples_tex.create_from_image(samples_img, Texture.FLAG_FILTER) self.samples_tex.create_from_image(samples_img, 0) #Texture.FLAG_FILTER)
var player := AudioStreamPlayer.new() # Make one for each channel, later var player := AudioStreamPlayer.new() # Make one for each channel, later

35
shaders/audio_renderer.gdshader
View File

@ -98,7 +98,7 @@ vec4 test_writeback(sampler2D tex, vec2 uv) {
// With the 258 texel header, which uses 3 texels of margin, 255 would be subtracted from the above payload, // With the 258 texel header, which uses 3 texels of margin, 255 would be subtracted from the above payload,
// leaving 261121 texels for the sample data. // leaving 261121 texels for the sample data.
const float HEADER_LENGTH_TEXELS = 6.0; const float HEADER_LENGTH_TEXELS = 8.0;
uniform sampler2D instrument_samples; uniform sampler2D instrument_samples;
uniform vec2 instrument_samples_size = vec2(2048.0, 128.0); uniform vec2 instrument_samples_size = vec2(2048.0, 128.0);
uniform float instrument_row_padding = 3.0; // In case we want to go to cubic filtering uniform float instrument_row_padding = 3.0; // In case we want to go to cubic filtering
@ -115,15 +115,21 @@ vec2 get_inst_texel(vec2 xy) {
return texture(instrument_samples, xy/instrument_samples_size).xw; return texture(instrument_samples, xy/instrument_samples_size).xw;
} }
float get_inst_texel_int16(float s) {
float x = mod(s, instrument_row_payload) + instrument_row_padding;
float y = trunc(s / instrument_row_payload);
return unpack_int16(texture(instrument_samples, (vec2(x, y) + 0.5)/instrument_samples_size).xw);
}
float get_instrument_sample(float instrument_index, float pitch_scale, float t, float t_end) { float get_instrument_sample(float instrument_index, float pitch_scale, float t, float t_end) {
// t_end is for ADSR purposes // t_end is for ADSR purposes
float header_offset = instrument_index * HEADER_LENGTH_TEXELS; float header_offset = instrument_index * HEADER_LENGTH_TEXELS;
float sample_start = unpack_uint16(get_inst_texel(vec2(header_offset, 0.0))); // The true start, after the prepended 3 frames of silence float sample_start = float(unpack_int32(vec4(get_inst_texel(vec2(header_offset, 0.0)), get_inst_texel(vec2(header_offset + 1.0, 0.0))))); // The true start, after the prepended 3 frames of silence
float sample_length = unpack_uint16(get_inst_texel(vec2(header_offset + 1.0, 0.0))); // 3 frames after the true end, because of how we loop float sample_length = unpack_uint16(get_inst_texel(vec2(header_offset + 2.0, 0.0))); // 3 frames after the true end, because of how we loop
float sample_loop_begin = unpack_uint16(get_inst_texel(vec2(header_offset + 2.0, 0.0))); // 3 frames after the true loop point float sample_loop_begin = unpack_uint16(get_inst_texel(vec2(header_offset + 3.0, 0.0))); // 3 frames after the true loop point
float sample_mixrate = unpack_uint16(get_inst_texel(vec2(header_offset + 3.0, 0.0))); float sample_mixrate = unpack_uint16(get_inst_texel(vec2(header_offset + 4.0, 0.0)));
vec2 attack_decay = get_inst_texel(vec2(header_offset + 4.0, 0.0)); vec2 attack_decay = get_inst_texel(vec2(header_offset + 5.0, 0.0));
vec2 sustain_release = get_inst_texel(vec2(header_offset + 5.0, 0.0)); vec2 sustain_release = get_inst_texel(vec2(header_offset + 6.0, 0.0));
// Calculate the point we want to sample in linear space // Calculate the point we want to sample in linear space
float mixrate = sample_mixrate * pitch_scale; float mixrate = sample_mixrate * pitch_scale;
float target_frame = t * mixrate; float target_frame = t * mixrate;
@ -134,10 +140,11 @@ float get_instrument_sample(float instrument_index, float pitch_scale, float t,
target_frame -= overshoot_loops*loop_length; target_frame -= overshoot_loops*loop_length;
// Now we need to identify the sampling point since our frames are spread across multiple rows for GPU reasons // Now we need to identify the sampling point since our frames are spread across multiple rows for GPU reasons
// We only sample from texel 4 onwards on a given row - texel 0 is the header, texels 1,2,3 are lead-in for filtering // We only sample from texel 4 onwards on a given row - texel 0 is the header, texels 1,2,3 are lead-in for filtering
// Note that y should be integral, but x should be continuous, as that's what applies the filtering!
target_frame += sample_start; target_frame += sample_start;
vec2 sample_xy = vec2(instrument_row_padding + mod(target_frame, instrument_row_payload), trunc(target_frame/instrument_row_payload)); float a = get_inst_texel_int16(floor(target_frame));
return rescale_int16(unpack_int16(get_inst_texel(sample_xy))); float b = get_inst_texel_int16(ceil(target_frame));
float mix_amount = fract(target_frame);
return rescale_int16(mix(a, b, mix_amount));
} }
const int NUM_CHANNELS = 8; const int NUM_CHANNELS = 8;
@ -196,13 +203,7 @@ vec4 render_song(int smp) {
} }
} }
// Convert the stereo float audio to S16LE // Convert the stereo float audio to S16LE
// return vec4(pack_float_to_int16(downmixed_stereo.x), pack_float_to_int16(downmixed_stereo.y)); return vec4(pack_float_to_int16(downmixed_stereo.x), pack_float_to_int16(downmixed_stereo.y));
// return vec4(pack_float_to_int16(downmixed_stereo.x), pack_float_to_int16(mod(t, 2.0) - 1.0));
vec2 isuv = vec2(mod(float(smp), instrument_samples_size.x), trunc(float(smp)/instrument_samples_size.x))/instrument_samples_size;
// float ins = rescale_int16(unpack_int16(texture(instrument_samples, isuv).xw));
// return vec4(pack_float_to_int16(ins), pack_float_to_int16(mod(t, 2.0) - 1.0));
return vec4(texture(instrument_samples, isuv).xw, pack_float_to_int16(mod(t, 2.0) - 1.0));
// return vec4(pack_float_to_int16((t/10.0) - 1.0), pack_float_to_int16(mod(t, 2.0) - 1.0));
} }
void fragment() { void fragment() {

4
test/audio_system.gd
View File

@ -138,9 +138,9 @@ func test_rendering() -> void:
var midi_events_bytes3 := StreamPeerBuffer.new() var midi_events_bytes3 := StreamPeerBuffer.new()
var midi_events_bytes4 := StreamPeerBuffer.new() var midi_events_bytes4 := StreamPeerBuffer.new()
for i in 2048: for i in 2048:
var t = i * 10.0 var t = i * 3.0
midi_events_bytes.put_32(t*32000) # t_start midi_events_bytes.put_32(t*32000) # t_start
midi_events_bytes2.put_32((t+3.0)*32000) # t_end midi_events_bytes2.put_32((t+2.75)*32000) # t_end
midi_events_bytes3.put_u8((i%35)) # instrument midi_events_bytes3.put_u8((i%35)) # instrument
midi_events_bytes3.put_u8(71) # pitch_idx midi_events_bytes3.put_u8(71) # pitch_idx
# midi_events_bytes.put_float((35 + (i%40))) # pitch_idx # midi_events_bytes.put_float((35 + (i%40))) # pitch_idx

1
test/audio_system.tscn
View File

@ -13,7 +13,6 @@ shader_param/instrument_row_payload = 2042.0
shader_param/reference_note = 71.0 shader_param/reference_note = 71.0
shader_param/output_mixrate = 32000.0 shader_param/output_mixrate = 32000.0
shader_param/midi_events_size = Vector2( 2048, 16 ) shader_param/midi_events_size = Vector2( 2048, 16 )
shader_param/t_scale = 524.0
[node name="audio_system" type="Node2D"] [node name="audio_system" type="Node2D"]
script = ExtResource( 1 ) script = ExtResource( 1 )