add useBeat

Author: takuma-hmng8

app/ShaderFx.tsx

@@ -12,8 +12,8 @@ export const ShaderFx = ({ children }: { children: React.ReactNode }) => {
          <PerformanceMonitor
             factor={1}
             onChange={({ factor }) => {
-               console.log(`dpr:${dpr}`);
-               // setDpr(Math.round((0.5 + 1.0 * factor) * 10) / 10);
+               // console.log(`dpr:${dpr}`);
+               setDpr(Math.round((0.5 + 1.0 * factor) * 10) / 10);
             }}>
             <Suspense fallback={null}>{children}</Suspense>
             <Perf position={"bottom-left"} minimal={false} />

app/playground/Playground.tsx

@@ -29,8 +29,7 @@ extend({ FxMaterial });
 export const Playground = () => {
    const ref = useRef<FxMaterialProps>();
 
-   const video = useVideoTexture("/glitch.mov");
-   const sample = useTexture("/test.jpg");
+   const video = useVideoTexture("/gorilla.mov");
 
    const { size, viewport } = useThree();
 
@@ -42,12 +41,12 @@ export const Playground = () => {
    set({
       texture: video,
       textureResolution: new THREE.Vector2(1920, 1080),
-      keyColor: new THREE.Color(0x000000),
-      similarity: 0.01,
-      spill: 0.1,
+      keyColor: new THREE.Color(0x71f998),
+      similarity: 0.2,
+      spill: 0.2,
       smoothness: 0.1,
-      contrast: 1.8,
-      gamma: 0.7,
+      contrast: 1,
+      gamma: 1,
       brightness: 0.0,
    });
 

app/useBeat/FxMaterial.tsx

@@ -0,0 +1,38 @@
+import * as THREE from "three";
+import { shaderMaterial } from "@react-three/drei";
+import fragment from "./main.frag";
+
+declare global {
+   namespace JSX {
+      interface IntrinsicElements {
+         fxMaterial: any;
+      }
+   }
+}
+
+export type FxMaterialProps = {
+   u_time: number;
+   u_resolution: THREE.Vector2;
+   u_fx: THREE.Texture;
+   u_noise: THREE.Texture;
+   u_noiseIntensity: number;
+};
+
+export const FxMaterial = shaderMaterial(
+   {
+      u_time: 0,
+      u_resolution: new THREE.Vector2(0, 0),
+      u_fx: new THREE.Texture(),
+      u_noise: new THREE.Texture(),
+      u_noiseIntensity: 0,
+   },
+
+   `
+		varying vec2 vUv;
+		void main() {
+			vUv = uv;
+			gl_Position = vec4(position, 1.0);
+		}
+	`,
+   fragment
+);

app/useBeat/Playground.tsx

@@ -0,0 +1,62 @@
+"use client";
+
+import * as THREE from "three";
+import { useEffect, useRef } from "react";
+import { useFrame, useThree, extend } from "@react-three/fiber";
+import {
+   useNoise,
+   useFluid,
+   useFxBlending,
+   useColorStrata,
+   useBrightnessPicker,
+   useChromaKey,
+   useBeat,
+   useFxTexture,
+} from "@/packages/use-shader-fx/src";
+import {
+   NoiseParams,
+   NOISE_PARAMS,
+} from "@/packages/use-shader-fx/src/hooks/useNoise";
+import {
+   ColorStrataParams,
+   COLORSTRATA_PARAMS,
+} from "@/packages/use-shader-fx/src/hooks/useColorStrata";
+import GUI from "lil-gui";
+import { useGUI } from "@/utils/useGUI";
+import { FxMaterial, FxMaterialProps } from "./FxMaterial";
+import { useTexture, useVideoTexture } from "@react-three/drei";
+import { Console } from "console";
+
+extend({ FxMaterial });
+
+export const Playground = () => {
+   const ref = useRef<FxMaterialProps>();
+
+   const { size, viewport } = useThree();
+   const video = useVideoTexture("/gorilla.mov");
+   const glitch = useVideoTexture("/glitch.mov");
+
+   const [updateNoise, set, { output: noise }] = useNoise({
+      size,
+      dpr: viewport.dpr,
+   });
+
+   const updateBeat = useBeat(157);
+
+   useFrame((props) => {
+      const { beat, fract, hash } = updateBeat(props.clock);
+      updateNoise(props, {
+         beat: beat,
+      });
+      ref.current!.u_noiseIntensity =
+         hash > 0.5 ? hash * fract : hash * fract * -1;
+      ref.current!.u_fx = hash > 0.5 ? glitch : video;
+   });
+
+   return (
+      <mesh>
+         <planeGeometry args={[2, 2]} />
+         <fxMaterial key={FxMaterial.key} u_noise={noise} ref={ref} />
+      </mesh>
+   );
+};

app/useBeat/main.frag

@@ -0,0 +1,17 @@
+precision highp float;
+varying vec2 vUv;
+uniform sampler2D u_fx;
+uniform sampler2D u_noise;
+uniform float u_noiseIntensity;
+
+void main() {
+	vec2 uv = vUv;
+
+	vec4 noise = texture2D(u_noise, uv);
+
+	uv += noise.rg * u_noiseIntensity;
+
+	vec4 fx = texture2D(u_fx, uv);
+
+	gl_FragColor = fx;
+}

app/useBeat/page.tsx

@@ -0,0 +1,20 @@
+import { ShaderFx } from "../ShaderFx";
+import { Playground } from "./Playground";
+
+export default function Page() {
+   return (
+      <div style={{ width: "100%" }}>
+         <div
+            style={{
+               position: "fixed",
+               width: "100%",
+               height: "100svh",
+               pointerEvents: "none",
+            }}>
+            <ShaderFx>
+               <Playground />
+            </ShaderFx>
+         </div>
+      </div>
+   );
+}

packages/use-shader-fx/src/hooks/useChromaKey/index.ts

@@ -15,11 +15,11 @@ export type ChromaKeyParams = {
    textureResolution?: THREE.Vector2;
    /** key color for chromakey processing , default: THREE.Color(0x00ff00) */
    keyColor?: THREE.Color;
-   /** If the similarity with the key color exceeds this value, it becomes transparent. , default: 0.01 */
+   /** If the similarity with the key color exceeds this value, it becomes transparent. , default: 0.2 */
    similarity?: number;
-   /** smoothness , default : 0.01 */
+   /** smoothness , default : 0.1 */
    smoothness?: number;
-   /** spill , default : 0.01 */
+   /** spill , default : 0.2 */
    spill?: number;
    /** tone correction , default : THREE.Vector4(1.0, 1.0, 1.0, 1.0) */
    color?: THREE.Vector4;
@@ -43,9 +43,9 @@ export const CHROMAKEY_PARAMS: ChromaKeyParams = {
    texture: new THREE.Texture(),
    textureResolution: new THREE.Vector2(0, 0),
    keyColor: new THREE.Color(0x00ff00),
-   similarity: 0.01,
-   smoothness: 0.01,
-   spill: 0.01,
+   similarity: 0.2,
+   smoothness: 0.1,
+   spill: 0.2,
    color: new THREE.Vector4(1.0, 1.0, 1.0, 1.0),
    contrast: 1.0,
    brightness: 0.0,

packages/use-shader-fx/src/hooks/useChromaKey/shader/main.frag

@@ -29,7 +29,7 @@ float getChromeDist(vec3 texColor){
 
 float getBoxFilteredChromaDist(vec3 rgb, vec2 uv)
 {
-	vec2 pixel_size = vec2(1.,1.);
+	vec2 pixel_size = vec2(1.) / u_textureResolution;
 	vec2 h_pixel_size = pixel_size / 2.0;
 	vec2 point_0 = vec2(pixel_size.x, h_pixel_size.y);
 	vec2 point_1 = vec2(h_pixel_size.x, -pixel_size.y);

packages/use-shader-fx/src/hooks/useNoise/index.ts

@@ -23,6 +23,8 @@ export type NoiseParams = {
    warpDirection?: THREE.Vector2;
    /** strength of domain warping , default:8.0 */
    warpStrength?: number;
+   /** you can get into the rhythm ♪ , default:null */
+   beat?: number | null;
 };
 
 export type NoiseObject = {
@@ -41,6 +43,7 @@ export const NOISE_PARAMS: NoiseParams = {
    warpOctaves: 2,
    warpDirection: new THREE.Vector2(2.0, 2.0),
    warpStrength: 8.0,
+   beat: null,
 };
 
 /**
@@ -80,7 +83,7 @@ export const useNoise = ({
          setUniform(material, "warpDirection", params.warpDirection!);
          setUniform(material, "warpStrength", params.warpStrength!);
 
-         setUniform(material, "uTime", clock.getElapsedTime());
+         setUniform(material, "uTime", params.beat ?? clock.getElapsedTime());
 
          return updateRenderTarget(gl);
       },

packages/use-shader-fx/src/index.js

@@ -27,3 +27,4 @@ export { usePointer } from "./utils/usePointer";
 export { useResolution } from "./utils/useResolution";
 export { useSingleFBO } from "./utils/useSingleFBO";
 export { useCopyTexture } from "./utils/useCopyTexture";
+export { useBeat } from "./utils/useBeat";