リアルタイムコンパイルのsound shader editor
考えてみたら、リアルタイムコンパイルのsound shader editorって無いですね。使いたい人がいるかもで、pythonのソースを載せる事にしました。ファイルの入出力は付いていません。ライブコーディングみたいな感じで使えます。ちょこちょこと音が変わるので面白いです。midiも使えます。m0~m15まで用意しました。0~1を割り当ててます。mix(440.,880.,mo0)
みたいな感じで使うのがいいかと。リアルタイムコンパイルは外せます。OpenGLなのでコンパイルが速いので、ストレスはあまりないと思いますが、チャンクとチャンクの間の書き込み時間にコンパイルをさせているので、重いshaderだと音が遅れることが有るかも。
GUIにflexxを使っています。エディターにCodeMirror.jsです。Ace.jsはflexxと相性が悪いみたいなのでCodeMirror.jsにしました。ただリンクを辿れるところにGLSLのシンタックスハイライトが無かったので使えませんでした。glsl.js
を自力で書くか、探してローカルに置けば使えます。
FFTのグラフは、良く解らなかったので、適当にそれっぽく使っているだけです。解る人がいたら教えてください。
shadertoyのmusic shaderをコピペで音が出ます。
お勧めは
https://www.shadertoy.com/user/athibaul
https://www.shadertoy.com/user/nabr
この2人です。
from OpenGL.GL import * from OpenGL.WGL import * from ctypes import * import numpy import time import rtmidi2 import threading import pyaudio import re from flexx import flx import os kernel32 = windll.kernel32 user32 = windll.user32 winmm = windll.winmm class Tick(): def __init__(self, chunk, rate): self.n = 0 self.chunk = chunk self.rate = rate self.startN = 0 self.endN = 600 def clucN(self, sec): return sec * self.rate / self.chunk def clucTime(self, n): return n * self.chunk / self.rate def startTime(self, sec): self.startN = self.clucN(sec) self.n = max(self.startN, self.n) def endTime(self, sec): self.endN = self.clucN(sec) self.n = min(self.endN, self.n) def reset(self): self.n = self.startN def time(self): return self.clucTime(self.n) def tick(self): self.n += 1 if self.endN < self.n: self.n = self.startN return self.n class ShaderSound: def __init__(self, **kwargs): self.chunk = 4096 self.rate = 48000 self.channels = 2 self.shader = """vec2 mainSound(int samp, float time) { return vec2( sin(6.2831*440.0*time)*exp(-3.0*time) ); } """ self.__dict__.update(kwargs) self.tick = Tick(self.chunk, self.rate) self.success = -1 self.compileLog='' self.chunkData=[] self.active = False self.flag = True self.volume = 1.0 self.midi = [0] * 16 self.time=0 self.startTime=0 self.endTime=180 self.head = """ #version 430 out vec2 gain; uniform float iFrameCount; uniform float m0; uniform float m1; uniform float m2; uniform float m3; uniform float m4; uniform float m5; uniform float m6; uniform float m7; uniform float m8; uniform float m9; uniform float m10; uniform float m11; uniform float m12; uniform float m13; uniform float m14; uniform float m15; const float iChunk = {0:.1f}; const float iSampleRate = {1:.1f}; """.format(self.chunk ,self.rate) + """ vec2 mainSound( int samp, float time ); void main(){ float time = (iChunk * iFrameCount + float(gl_VertexID)) / iSampleRate; gain = clamp(mainSound(int(iSampleRate),time), -1.0, 1.0); } """ def compileProgram(self,src): shader = glCreateShader(GL_VERTEX_SHADER) glShaderSource(shader, self.head + src) glCompileShader(shader) if glGetShaderiv(shader, GL_COMPILE_STATUS) != GL_TRUE: self.compileLog=glGetShaderInfoLog(shader).decode() glDeleteShader(shader) return -1 else: self.compileLog='Success' program = glCreateProgram() glAttachShader(program, shader) glDeleteShader(shader) outs = cast((c_char_p*1)(b"gain"), POINTER(POINTER(c_char))) glTransformFeedbackVaryings(program, 1, outs, GL_INTERLEAVED_ATTRIBS) glLinkProgram(program) return program def glDraw(self): hWnd = user32.CreateWindowExA(0,0xC018,0,0,0,0,0,0,0,0,0,0) hDC = user32.GetDC(hWnd) pfd = PIXELFORMATDESCRIPTOR(0,1,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,0,0) SetPixelFormat(hDC,ChoosePixelFormat(hDC, pfd), pfd) hGLrc = wglCreateContext(hDC) wglMakeCurrent(hDC, hGLrc) self.samples = (c_float * self.chunk * 2)() vbo = glGenBuffers(1) glBindBuffer(GL_ARRAY_BUFFER, vbo) glBufferData(GL_ARRAY_BUFFER, sizeof(self.samples), None, GL_STATIC_DRAW) glBindBufferBase(GL_TRANSFORM_FEEDBACK_BUFFER, 0, vbo) p = pyaudio.PyAudio() stream = p.open( format = pyaudio.paFloat32, channels = 2, rate = self.rate, frames_per_buffer = self.chunk, output=True, input=False ) stream.start_stream() self.active = True pMain = self.compileProgram("vec2 mainSound(int samp,float time){return vec2(0);}") while self.active: time.sleep(0.01) if self.flag is False: p=self.compileProgram(self.shader) self.success = p if p>-1: tmp=pMain pMain=p glDeleteProgram(tmp) self.flag = True glUseProgram(pMain) glUniform1f(glGetUniformLocation(pMain, "iFrameCount"), self.tick.tick()) for i in range(16): glUniform1f(glGetUniformLocation(pMain, "m{0}".format(i)), self.midi[i]); glEnable(GL_RASTERIZER_DISCARD) glBeginTransformFeedback(GL_POINTS) glDrawArrays(GL_POINTS, 0, self.chunk) glEndTransformFeedback() glDisable(GL_RASTERIZER_DISCARD) glGetBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(self.samples), byref(self.samples)) data=numpy.frombuffer(self.samples, dtype=numpy.float32) self.chunkData=data stream.write((data*self.volume).tobytes()) stream.stop_stream() stream.close() wglMakeCurrent(0, 0) wglDeleteContext(hGLrc) user32.ReleaseDC(hWnd, hDC) user32.PostQuitMessage(0) user32.DestroyWindow(hWnd) def run(self): t = threading.Thread(target=self.glDraw) t.start() def stop(self): self.active=False def compile(self,src): self.shader=src self.flag = False def getSuccess(self): return self.success def getShader(self): return self.shader def getCompileLog(self): while self.flag is False: time.sleep(0.01) return self.compileLog def getTime(self): return self.tick.time() def getChunkData(self): return self.chunkData def reTime(self): self.tick.reset() def setTimeBand(self,start,end): self.tick.startTime(start) self.tick.endTime(end) def getTimeBand(self): return dict( start_time = self.startTime, end_time = self.endTime ) def setMidiValue(self, idx, value): if idx>-1 and idx<16: self.midi[idx]=value def setVolume(self, volume): self.volume=volume # web editorは CodeMirrorを使ってます。 https://codemirror.net/ base_url = 'https://cdnjs.cloudflare.com/ajax/libs/codemirror/' flx.assets.associate_asset(__name__, base_url + '5.21.0/codemirror.min.css') flx.assets.associate_asset(__name__, base_url + '5.21.0/codemirror.min.js') flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/midnight.css') flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/rubyblue.css') flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/erlang-dark.css') flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/cobalt.css') # ローカルにシンタックスハイライトのスクリプトを書いておく場合。 #with open(os.getcwd() + '/glsl.js') as f:glsl = f.read() #flx.assets.associate_asset(__name__, 'glsl.js', glsl) class Editor(flx.Widget): def init(self): global window options = dict( #mode='glsl', theme='cobalt', autofocus=True, styleActiveLine=True, matchBrackets=True, tabSize=2, lineWrapping=True, lineNumbers=True, ) self.cm = window.CodeMirror(self.node, options) self.cm.on("change", self.change); @flx.reaction('size') def __on_size(self, *events): self.cm.refresh() @flx.action def setValue(self, value): self.cm.setValue(value) @flx.emitter def change(self): self.emit('clear',{}) for s in self.cm.getValue().split('\n'): self.emit('value',dict(line=s)) return {} class WavePlot(flx.CanvasWidget): def init(self): super().init() self.ctx = self.node.getContext('2d') self.tick() def tick(self): global window self.emit('waveUpdate', {}) window.setTimeout(self.tick, 100) @flx.action def wave(self, data): w, h = self.size size = len(data) pitch = w/size rate = h/2 self.ctx.clearRect(0, 0, w, h) self.ctx.beginPath() self.ctx.strokeStyle = '#f80' self.ctx.lineWidth = 1 self.ctx.lineCap = 'round' self.ctx.moveTo(0,(data[0]+1)*h/2*.7) for i in range(size-1): self.ctx.lineTo(pitch*(i+1),(data[i+1]+1)*h/2*.7) self.ctx.stroke() self.ctx.closePath() class FftPlot(flx.CanvasWidget): def init(self): super().init() self.ctx = self.node.getContext('2d') self.tick() def tick(self): global window self.emit('fftUpdate', {}) window.setTimeout(self.tick, 100) @flx.action def wave(self, data): w, h = self.size size = len(data) pitch = w/size rate = h/2 self.ctx.clearRect(0, 0, w, h) self.ctx.beginPath() self.ctx.strokeStyle = '#f80' self.ctx.lineWidth = 1 self.ctx.lineCap = 'round' self.ctx.moveTo(0,(data[0]+1)*h/2) for i in range(size-1): self.ctx.lineTo(pitch*(i+1),(data[i+1]+1)*h/2) self.ctx.stroke() self.ctx.closePath() class Page(flx.Widget): CSS = """ body { background: rgb(15,20,30); font-size: 14px; } .CodeMirror { font-family: Monaco, 'Andale Mono', 'Lucida Console', 'Bitstream Vera Sans Mono', 'Courier New', Courier, monospace; font-size: 9pt; width: 100%; height: 100%; } """ def init(self): with flx.VBox(flex=1): with flx.HBox(flex=1): self.editor=Editor(flex=1) with flx.VBox(flex=0, maxsize=220): flx.Widget(flex=1,minsize=10) self.plot = WavePlot(flex=1,minsize=100) self.fft_plot = FftPlot(flex=1,minsize=100) flx.Label(flex=0,text='Volume', style='color:whitesmoke') self.volume = flx.Slider(flex=0,min=0,max=1,value=1) with flx.HBox(flex=0): flx.Label(text='start time', style='color:whitesmoke') flx.Widget(flex=1) self.startTime = flx.LineEdit(maxsize=50,title='start time:', text='0',) with flx.HBox(flex=0): flx.Label(text='end time', style='color:whitesmoke') flx.Widget(flex=1) self.endTime = flx.LineEdit(maxsize=50,title='end time:', text='180') with flx.HBox(flex=0): flx.Widget(flex=1) self.timereset_button = flx.Button(flex=0,text='TimeReset') with flx.HBox(flex=0): self.auto = flx.CheckBox(checked=True,text='Auto Compile', style='color:whitesmoke') flx.Widget(flex=1) self.compile_button = flx.Button(text='_') with flx.HBox(flex=0): flx.Widget(minsize=15) self.labelTime = flx.Label(flex=0,style='color:orange') flx.Widget(flex=1) with flx.HBox(flex=0): flx.Widget(flex=0,minsize=10) self.labelDebag = flx.Label(flex=1,text="", style='border:1px solid red; color:whitesmoke') flx.Widget(flex=0,minsize=10) flx.Widget(flex=1) self.tick() def tick(self): global window self.emit('cmd', dict(cmd = 'self.timeUpate()')) window.setTimeout(self.tick, 100) @flx.action def timeUpate(self, s): self.labelTime.set_text("Time : {0:.1f}sec".format(s)) @flx.action def setValue(self, value): self.editor.setValue(value) @flx.action def waveUpdate(self, d): self.plot.wave(d) @flx.action def fftUpdate(self, d): self.fft_plot.wave(d) @flx.action def debag(self, s): self.labelDebag.set_text(s) @flx.reaction('timereset_button.pointer_click') def _6(self, *events): self.emit('cmd', dict(cmd = 'mzk.reTime()')) @flx.reaction('startTime.user_done') def _7(self, *events): if self.startTime.text.isdigit(): if int(self.startTime.text)>=int(self.endTime.text): self.startTime.set_text(str(int(self.endTime.text)-1)) else: self.startTime.set_text(str(int(self.endTime.text)-1)) self.emit('cmd', dict(cmd = 'mzk.setTimeBand({0},{1})'.format(self.startTime.text, self.endTime.text))) self.emit('cmd', dict(cmd = 'mzk.reTime()')) @flx.reaction('endTime.user_done') def _8(self, *events): if self.endTime.text.isdigit(): if int(self.startTime.text)>= int(self.endTime.text): self.endTime.set_text(str(int(self.startTime.text)+1)) else: self.endTime.set_text(str(int(self.startTime.text)+1)) self.emit('cmd', dict(cmd = 'mzk.setTimeBand({0},{1})'.format(self.startTime.text, self.endTime.text))) self.emit('cmd', dict(cmd = 'mzk.reTime()')) @flx.reaction('auto.pointer_click' ) def _9(self, *events): if not self.auto.checked: self.compile_button.set_text('Compile') else: self.compile_button.set_text('_') @flx.reaction('compile_button.pointer_click' ) def _10(self, *events): if not self.auto.checked: self.emit('cmd', dict(cmd = "self.compile()")) @flx.reaction('editor.change') def _11(self, *events): if self.auto.checked: self.emit('cmd', dict(cmd = "self.compile()")) @flx.reaction('!editor.value') def _12(self, *events): for ev in events: self.emit('value', dict(line = ev['line'])) @flx.reaction('!plot.waveUpdate') def _13(self, *events): ev = events[-1] self.emit('cmd', dict(cmd = "self.waveUpdate()")) @flx.reaction('!fft_plot.fftUpdate') def _14(self, *events): ev = events[-1] self.emit('cmd', dict(cmd = "self.fftUpdate()")) @flx.reaction('volume.user_done') def _15(self, *events): self.emit('cmd', dict(cmd = "mzk.setVolume("+self.volume.value+")")) @flx.reaction('!editor.clear') def _16(self, *events): self.emit('cmd', dict(cmd = "self.valueClear()")) class App(flx.PyComponent): def init(self): global mzk self.widget = Page() self.setValue() self.value='' # midi def callback(message, time_stamp): a,idx,value =message # KORG nano KONTROL2 の場合 idx=(idx//16)*8+(idx%16) value=value/127 mzk.setMidiValue(idx, value) self.midi_in = rtmidi2.MidiIn() self.midi_in.callback = callback if len(rtmidi2.get_in_ports())>0: self.midi_in.open_port(0) @flx.reaction('!widget.cmd') def _cmd(self, *events): global mzk for ev in events: eval(ev['cmd']) def timeUpate(self): global mzk self.widget.timeUpate(mzk.getTime()) def waveUpdate(self): global mzk d=mzk.getChunkData() w=(d[::2]+d[1::2])/2 self.widget.waveUpdate(w) def fftUpdate(self): # http://abcz.wp.xdomain.jp/2020/05/18/numpy-fft/ # fftが良くわからないから、それっぽくしただけ。 global mzk d=mzk.getChunkData() w=(d[::2]+d[1::2])/2 w=numpy.abs(numpy.fft.rfft(w)) self.widget.fftUpdate(w) @flx.reaction('!widget.value') def s14(self, *events): for ev in events: self.value += ev['line'] + '\n' def setValue(self): global mzk self.widget.setValue(mzk.getShader()) def valueClear(self): self.value='' def compile(self): if len(self.value)==0: return mzk.compile(self.value) self.value='' self.widget.debag(mzk.getCompileLog()) if __name__ == '__main__': src = """vec2 mainSound(int samp, float time) { time=mod(time,2.); return vec2( sin(6.2831*440.0*time)*exp(-3.0*time) ); } """ mzk=ShaderSound(shader=src) #mzk=ShaderSound() mzk.run() flx.launch( App, title='MzkShaderEdtor', size=(800, 500), pos=(500, 10) ) flx.run() mzk.stop()