テラByteの時代にキロByte

shader又はdemosceneに関係する事

リアルタイムコンパイルのsound shader editor

考えてみたら、リアルタイムコンパイルのsound shader editorって無いですね。使いたい人がいるかもで、pythonのソースを載せる事にしました。ファイルの入出力は付いていません。ライブコーディングみたいな感じで使えます。ちょこちょこと音が変わるので面白いです。midiも使えます。m0~m15まで用意しました。0~1を割り当ててます。mix(440.,880.,mo0)みたいな感じで使うのがいいかと。リアルタイムコンパイルは外せます。OpenGLなのでコンパイルが速いので、ストレスはあまりないと思いますが、チャンクとチャンクの間の書き込み時間にコンパイルをさせているので、重いshaderだと音が遅れることが有るかも。
GUIにflexxを使っています。エディターにCodeMirror.jsです。Ace.jsはflexxと相性が悪いみたいなのでCodeMirror.jsにしました。ただリンクを辿れるところにGLSLのシンタックスハイライトが無かったので使えませんでした。glsl.jsを自力で書くか、探してローカルに置けば使えます。
FFTのグラフは、良く解らなかったので、適当にそれっぽく使っているだけです。解る人がいたら教えてください。
shadertoyのmusic shaderをコピペで音が出ます。
お勧めは
https://www.shadertoy.com/user/athibaul
https://www.shadertoy.com/user/nabr
この2人です。

from OpenGL.GL import *
from OpenGL.WGL import *
from ctypes import *
import numpy
import time
import rtmidi2
import threading
import pyaudio
import re
from flexx import flx
import os

kernel32 = windll.kernel32
user32 = windll.user32
winmm = windll.winmm

class Tick():
    def __init__(self, chunk, rate):
        self.n = 0
        self.chunk = chunk
        self.rate = rate
        self.startN = 0
        self.endN = 600
        
    def clucN(self, sec):
        return  sec * self.rate / self.chunk
        
    def clucTime(self, n):
        return  n * self.chunk / self.rate
        
    def startTime(self, sec):
        self.startN = self.clucN(sec)
        self.n = max(self.startN, self.n)

    def endTime(self, sec):
        self.endN = self.clucN(sec)
        self.n = min(self.endN, self.n)
        
    def reset(self):
        self.n = self.startN

    def time(self):
        return self.clucTime(self.n)
    
    def tick(self):
        self.n += 1
        if self.endN < self.n:
            self.n = self.startN
        return self.n

class ShaderSound:
    def __init__(self, **kwargs):
        self.chunk = 4096
        self.rate = 48000
        self.channels = 2
        self.shader = """vec2 mainSound(int samp, float time)
{
    return vec2( sin(6.2831*440.0*time)*exp(-3.0*time) );
}
"""
        self.__dict__.update(kwargs)

        self.tick = Tick(self.chunk, self.rate)
        self.success = -1
        self.compileLog=''
        self.chunkData=[]
        self.active = False
        self.flag = True
        self.volume = 1.0
        self.midi = [0] * 16
        self.time=0
        self.startTime=0
        self.endTime=180
        self.head = """
#version 430
out vec2 gain; 
uniform float iFrameCount;
uniform float m0;
uniform float m1;
uniform float m2;
uniform float m3;
uniform float m4;
uniform float m5;
uniform float m6;
uniform float m7;
uniform float m8;
uniform float m9;
uniform float m10;
uniform float m11;
uniform float m12;
uniform float m13;
uniform float m14;
uniform float m15;
const float iChunk = {0:.1f};
const float iSampleRate = {1:.1f};
""".format(self.chunk ,self.rate) + """
vec2 mainSound( int samp, float time );
void main(){
    float time = (iChunk * iFrameCount + float(gl_VertexID)) / iSampleRate;
    gain = clamp(mainSound(int(iSampleRate),time), -1.0, 1.0);
}
"""

    def compileProgram(self,src):
        shader = glCreateShader(GL_VERTEX_SHADER)
        glShaderSource(shader, self.head + src)
        glCompileShader(shader)
        if glGetShaderiv(shader, GL_COMPILE_STATUS) != GL_TRUE:
            self.compileLog=glGetShaderInfoLog(shader).decode()
            glDeleteShader(shader)
            return -1
        else:
            self.compileLog='Success'
        program = glCreateProgram()
        glAttachShader(program, shader)
        glDeleteShader(shader)
        outs = cast((c_char_p*1)(b"gain"), POINTER(POINTER(c_char)))
        glTransformFeedbackVaryings(program, 1, outs, GL_INTERLEAVED_ATTRIBS)
        glLinkProgram(program)
        return program


    def glDraw(self):
        hWnd = user32.CreateWindowExA(0,0xC018,0,0,0,0,0,0,0,0,0,0)
        hDC = user32.GetDC(hWnd)
        pfd = PIXELFORMATDESCRIPTOR(0,1,0,32,0,0,0,0,0,0,0,0,0,0,0,0,0,32,0,0,0,0,0,0,0)
        SetPixelFormat(hDC,ChoosePixelFormat(hDC, pfd), pfd)
        hGLrc = wglCreateContext(hDC)
        wglMakeCurrent(hDC, hGLrc)
        self.samples = (c_float * self.chunk * 2)()
        vbo = glGenBuffers(1)
        glBindBuffer(GL_ARRAY_BUFFER, vbo)
        glBufferData(GL_ARRAY_BUFFER, sizeof(self.samples), None, GL_STATIC_DRAW)
        glBindBufferBase(GL_TRANSFORM_FEEDBACK_BUFFER, 0, vbo)
        p = pyaudio.PyAudio()
        stream = p.open(
            format = pyaudio.paFloat32,
            channels = 2,
            rate = self.rate,
            frames_per_buffer = self.chunk,
            output=True,
            input=False
        )
        stream.start_stream()
        self.active = True
        pMain = self.compileProgram("vec2 mainSound(int samp,float time){return vec2(0);}")
        while self.active:
            time.sleep(0.01)
            if self.flag is False:
                p=self.compileProgram(self.shader)
                self.success = p
                if p>-1:
                    tmp=pMain
                    pMain=p
                    glDeleteProgram(tmp)
                self.flag = True
            glUseProgram(pMain)
            glUniform1f(glGetUniformLocation(pMain, "iFrameCount"), self.tick.tick())
            for i in range(16):
                glUniform1f(glGetUniformLocation(pMain, "m{0}".format(i)), self.midi[i]);
            glEnable(GL_RASTERIZER_DISCARD)
            glBeginTransformFeedback(GL_POINTS)
            glDrawArrays(GL_POINTS, 0, self.chunk)
            glEndTransformFeedback()
            glDisable(GL_RASTERIZER_DISCARD)
            glGetBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(self.samples), byref(self.samples))
            data=numpy.frombuffer(self.samples, dtype=numpy.float32)
            self.chunkData=data
            stream.write((data*self.volume).tobytes())
        stream.stop_stream()
        stream.close()
        wglMakeCurrent(0, 0)
        wglDeleteContext(hGLrc)
        user32.ReleaseDC(hWnd, hDC)
        user32.PostQuitMessage(0)
        user32.DestroyWindow(hWnd)
    
    def run(self):
        t = threading.Thread(target=self.glDraw)
        t.start()
    
    def stop(self):
        self.active=False

        
    def compile(self,src):
        self.shader=src
        self.flag = False
    
    def getSuccess(self):
        return self.success
        
    def getShader(self):
        return self.shader
        
    def getCompileLog(self):
        while self.flag is False:
            time.sleep(0.01)
        return self.compileLog

    def getTime(self):
        return self.tick.time()

    def getChunkData(self):
        return self.chunkData
        
    def reTime(self):
        self.tick.reset()
        
    def setTimeBand(self,start,end):
        self.tick.startTime(start)
        self.tick.endTime(end)

    def getTimeBand(self):
        return dict(
            start_time = self.startTime,
            end_time = self.endTime
        )

    def setMidiValue(self, idx, value):
        if idx>-1 and idx<16:
                self.midi[idx]=value
                
    def setVolume(self, volume):
        self.volume=volume
                
# web editorは CodeMirrorを使ってます。  https://codemirror.net/
base_url = 'https://cdnjs.cloudflare.com/ajax/libs/codemirror/'
flx.assets.associate_asset(__name__, base_url + '5.21.0/codemirror.min.css')
flx.assets.associate_asset(__name__, base_url + '5.21.0/codemirror.min.js')
flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/midnight.css')
flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/rubyblue.css')
flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/erlang-dark.css')
flx.assets.associate_asset(__name__, base_url + '5.21.0/theme/cobalt.css')

# ローカルにシンタックスハイライトのスクリプトを書いておく場合。
#with open(os.getcwd() + '/glsl.js') as f:glsl = f.read()
#flx.assets.associate_asset(__name__, 'glsl.js', glsl)

class Editor(flx.Widget):
    def init(self):
        global window
        options = dict(
            #mode='glsl',
            theme='cobalt',
            autofocus=True,
            styleActiveLine=True,
            matchBrackets=True,
            tabSize=2,
            lineWrapping=True,
            lineNumbers=True,
        )
        self.cm = window.CodeMirror(self.node, options)
        self.cm.on("change", self.change);

    @flx.reaction('size')
    def __on_size(self, *events):
        self.cm.refresh()

    @flx.action
    def setValue(self, value):
        self.cm.setValue(value)
        
    @flx.emitter
    def change(self):
        self.emit('clear',{})
        for s in self.cm.getValue().split('\n'):
            self.emit('value',dict(line=s))
        return {}

class WavePlot(flx.CanvasWidget):
    def init(self):
        super().init()
        self.ctx = self.node.getContext('2d')
        self.tick()
        
    def tick(self):
        global window
        self.emit('waveUpdate', {})
        window.setTimeout(self.tick, 100)

    @flx.action
    def wave(self, data):
        w, h = self.size
        size = len(data)
        pitch = w/size
        rate = h/2
        self.ctx.clearRect(0, 0, w, h)
        self.ctx.beginPath()
        self.ctx.strokeStyle = '#f80'
        self.ctx.lineWidth = 1
        self.ctx.lineCap = 'round'
        self.ctx.moveTo(0,(data[0]+1)*h/2*.7)
        for i in range(size-1): 
            self.ctx.lineTo(pitch*(i+1),(data[i+1]+1)*h/2*.7)
        self.ctx.stroke()
        self.ctx.closePath()

class FftPlot(flx.CanvasWidget):
    def init(self):
        super().init()
        self.ctx = self.node.getContext('2d')
        self.tick()
        
    def tick(self):
        global window
        self.emit('fftUpdate', {})
        window.setTimeout(self.tick, 100)

    @flx.action
    def wave(self, data):
        w, h = self.size
        size = len(data)
        pitch = w/size
        rate = h/2
        self.ctx.clearRect(0, 0, w, h)
        self.ctx.beginPath()
        self.ctx.strokeStyle = '#f80'
        self.ctx.lineWidth = 1
        self.ctx.lineCap = 'round'
        self.ctx.moveTo(0,(data[0]+1)*h/2)
        for i in range(size-1): 
            self.ctx.lineTo(pitch*(i+1),(data[i+1]+1)*h/2)
        self.ctx.stroke()
        self.ctx.closePath()


class Page(flx.Widget):
    CSS = """
    body {
        background: rgb(15,20,30);
        font-size: 14px;
    }

    .CodeMirror {
        font-family: Monaco, 'Andale Mono', 'Lucida Console', 'Bitstream Vera Sans Mono', 'Courier New', Courier, monospace;
        font-size: 9pt;
        width: 100%;
        height: 100%;
    }
    """

    def init(self):
        with flx.VBox(flex=1):
            with flx.HBox(flex=1):
                self.editor=Editor(flex=1)
                with flx.VBox(flex=0, maxsize=220):
                    flx.Widget(flex=1,minsize=10)
                    self.plot = WavePlot(flex=1,minsize=100)
                    self.fft_plot = FftPlot(flex=1,minsize=100)
                    flx.Label(flex=0,text='Volume', style='color:whitesmoke')
                    self.volume = flx.Slider(flex=0,min=0,max=1,value=1)
                    with flx.HBox(flex=0):
                        flx.Label(text='start time', style='color:whitesmoke')
                        flx.Widget(flex=1)
                        self.startTime = flx.LineEdit(maxsize=50,title='start time:', text='0',)
                    with flx.HBox(flex=0):
                        flx.Label(text='end time', style='color:whitesmoke')
                        flx.Widget(flex=1)
                        self.endTime = flx.LineEdit(maxsize=50,title='end time:', text='180')
                    with flx.HBox(flex=0):
                        flx.Widget(flex=1)
                        self.timereset_button = flx.Button(flex=0,text='TimeReset')
                    with flx.HBox(flex=0):
                        self.auto = flx.CheckBox(checked=True,text='Auto Compile', style='color:whitesmoke')
                        flx.Widget(flex=1)
                        self.compile_button = flx.Button(text='_')
            with flx.HBox(flex=0):
                flx.Widget(minsize=15)
                self.labelTime = flx.Label(flex=0,style='color:orange')
                flx.Widget(flex=1)
            with flx.HBox(flex=0):
                flx.Widget(flex=0,minsize=10)
                self.labelDebag = flx.Label(flex=1,text="",  style='border:1px solid red; color:whitesmoke')
                flx.Widget(flex=0,minsize=10)
            flx.Widget(flex=1)
        self.tick()
        
    def tick(self):
        global window
        self.emit('cmd', dict(cmd = 'self.timeUpate()'))
        window.setTimeout(self.tick, 100)

    @flx.action
    def timeUpate(self, s):
        self.labelTime.set_text("Time : {0:.1f}sec".format(s))

    @flx.action
    def setValue(self, value):
        self.editor.setValue(value)
                
    @flx.action
    def waveUpdate(self, d):
        self.plot.wave(d)

    @flx.action
    def fftUpdate(self, d):
        self.fft_plot.wave(d)

    @flx.action
    def debag(self, s):
        self.labelDebag.set_text(s)

    @flx.reaction('timereset_button.pointer_click')
    def _6(self, *events):
        self.emit('cmd', dict(cmd = 'mzk.reTime()'))

    @flx.reaction('startTime.user_done')
    def _7(self, *events):
        if self.startTime.text.isdigit():
            if int(self.startTime.text)>=int(self.endTime.text):
                self.startTime.set_text(str(int(self.endTime.text)-1))
        else:
            self.startTime.set_text(str(int(self.endTime.text)-1))
        self.emit('cmd', dict(cmd = 'mzk.setTimeBand({0},{1})'.format(self.startTime.text, self.endTime.text)))
        self.emit('cmd', dict(cmd = 'mzk.reTime()'))

    @flx.reaction('endTime.user_done')
    def _8(self, *events):
        if self.endTime.text.isdigit():
            if int(self.startTime.text)>= int(self.endTime.text):
                 self.endTime.set_text(str(int(self.startTime.text)+1))
        else:
           self.endTime.set_text(str(int(self.startTime.text)+1))
        self.emit('cmd', dict(cmd = 'mzk.setTimeBand({0},{1})'.format(self.startTime.text, self.endTime.text)))
        self.emit('cmd', dict(cmd = 'mzk.reTime()'))

    @flx.reaction('auto.pointer_click' )
    def _9(self, *events):
        if not self.auto.checked:
            self.compile_button.set_text('Compile')
        else:
            self.compile_button.set_text('_')

    @flx.reaction('compile_button.pointer_click' )
    def _10(self, *events):
        if not self.auto.checked:
            self.emit('cmd', dict(cmd = "self.compile()"))

    @flx.reaction('editor.change')
    def _11(self, *events):
        if self.auto.checked:
            self.emit('cmd', dict(cmd = "self.compile()"))

    @flx.reaction('!editor.value')
    def _12(self, *events):
        for ev in events:
            self.emit('value', dict(line = ev['line']))
            
    @flx.reaction('!plot.waveUpdate')
    def _13(self, *events):
        ev = events[-1]
        self.emit('cmd', dict(cmd = "self.waveUpdate()"))
        
    @flx.reaction('!fft_plot.fftUpdate')
    def _14(self, *events):
        ev = events[-1]
        self.emit('cmd', dict(cmd = "self.fftUpdate()"))
        
    @flx.reaction('volume.user_done')
    def _15(self, *events):
        self.emit('cmd', dict(cmd = "mzk.setVolume("+self.volume.value+")"))

    @flx.reaction('!editor.clear')
    def _16(self, *events):
        self.emit('cmd', dict(cmd = "self.valueClear()"))

class App(flx.PyComponent):
    def init(self):
        global mzk
        self.widget = Page()
        self.setValue()
        self.value=''

        # midi
        def callback(message, time_stamp):
            a,idx,value =message
            # KORG nano KONTROL2 の場合
            idx=(idx//16)*8+(idx%16)
            value=value/127
            mzk.setMidiValue(idx, value)

        self.midi_in = rtmidi2.MidiIn()
        self.midi_in.callback = callback
        if len(rtmidi2.get_in_ports())>0:
            self.midi_in.open_port(0)

    @flx.reaction('!widget.cmd')
    def _cmd(self, *events):
        global mzk
        for ev in events:
            eval(ev['cmd'])

    def timeUpate(self):
        global mzk
        self.widget.timeUpate(mzk.getTime())

    def waveUpdate(self):
        global mzk
        d=mzk.getChunkData()
        w=(d[::2]+d[1::2])/2
        self.widget.waveUpdate(w)

    def fftUpdate(self):
        # http://abcz.wp.xdomain.jp/2020/05/18/numpy-fft/
        # fftが良くわからないから、それっぽくしただけ。
        global mzk
        d=mzk.getChunkData()
        w=(d[::2]+d[1::2])/2
        w=numpy.abs(numpy.fft.rfft(w))
        self.widget.fftUpdate(w)

    @flx.reaction('!widget.value')
    def s14(self, *events):
        for ev in events:
           self.value += ev['line'] + '\n'

    def setValue(self):
        global mzk
        self.widget.setValue(mzk.getShader())
        
    def valueClear(self):
        self.value=''

    def compile(self):
        if len(self.value)==0: return
        mzk.compile(self.value)
        self.value=''
        self.widget.debag(mzk.getCompileLog())
    
if __name__ == '__main__':

    src = """vec2 mainSound(int samp, float time)
{
    time=mod(time,2.);
    return vec2( sin(6.2831*440.0*time)*exp(-3.0*time) );
}
    """
    
    mzk=ShaderSound(shader=src)
    #mzk=ShaderSound()
    mzk.run()
    flx.launch(
        App,
        title='MzkShaderEdtor',
        size=(800, 500),
        pos=(500, 10)
    )
    flx.run()
    mzk.stop()