Skip to content

Commit abd2886

Browse files
author
takuma-hmng8
committed
useChromakey
1 parent 7d10bc2 commit abd2886

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+145
-4268
lines changed

‎app/playground/FxMaterial.tsx‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import * as THREE from "three";
2+
import { shaderMaterial } from "@react-three/drei";
3+
import fragment from "./main.frag";
4+
5+
declare global {
6+
namespace JSX {
7+
interface IntrinsicElements {
8+
fxMaterial: any;
9+
}
10+
}
11+
}
12+
13+
export type FxMaterialProps = {
14+
u_time: number;
15+
u_resolution: THREE.Vector2;
16+
u_textureResolution: THREE.Vector2;
17+
u_texture: THREE.Texture;
18+
u_keyColor: THREE.Color;
19+
};
20+
21+
export const FxMaterial = shaderMaterial(
22+
{
23+
u_time: 0,
24+
u_resolution: new THREE.Vector2(0, 0),
25+
u_textureResolution: new THREE.Vector2(0, 0),
26+
u_texture: null,
27+
u_keyColor: new THREE.Color(0x00ff00),
28+
},
29+
30+
`
31+
varying vec2 vUv;
32+
void main() {
33+
vUv = uv;
34+
gl_Position = vec4(position, 1.0);
35+
}
36+
`,
37+
fragment
38+
);

‎app/playground/Playground.tsx‎

Lines changed: 19 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
"use client";
22

33
import * as THREE from "three";
4-
import { useRef } from "react";
5-
import { useFrame, useThree } from "@react-three/fiber";
4+
import { useEffect, useRef } from "react";
5+
import { useFrame, useThree, extend } from "@react-three/fiber";
66
import {
77
useNoise,
88
useFluid,
@@ -20,132 +20,35 @@ import {
2020
} from "@/packages/use-shader-fx/src/hooks/useColorStrata";
2121
import GUI from "lil-gui";
2222
import { useGUI } from "@/utils/useGUI";
23+
import { FxMaterial, FxMaterialProps } from "./FxMaterial";
24+
import { useVideoTexture } from "@react-three/drei";
2325

24-
const CONFIG = {
25-
colorStrata: {
26-
...structuredClone(COLORSTRATA_PARAMS),
27-
laminateLayer: 10,
28-
laminateInterval: new THREE.Vector2(0.1, 0.1),
29-
laminateDetail: new THREE.Vector2(0.7, 0.7),
30-
distortion: new THREE.Vector2(10.0, 10.0),
31-
colorFactor: new THREE.Vector3(1, 1, 1),
32-
timeStrength: new THREE.Vector2(1, 1),
33-
noiseStrength: new THREE.Vector2(1, 1),
34-
} as ColorStrataParams,
35-
color: {
36-
color0: new THREE.Color(0xff0000),
37-
color1: new THREE.Color(0x0000ff),
38-
color2: new THREE.Color(0x00ff00),
39-
color3: new THREE.Color(0xffff00),
40-
},
41-
};
42-
43-
const setGUI = (gui: GUI) => {
44-
//color strata
45-
const colorStrata = gui.addFolder("colorStrata");
46-
colorStrata.add(CONFIG.colorStrata, "laminateLayer", 0, 20, 1);
47-
colorStrata.add(CONFIG.colorStrata, "scale", 0, 1, 0.01);
48-
const laminateInterval = colorStrata.addFolder("laminateInterval");
49-
laminateInterval.add(CONFIG.colorStrata.laminateInterval!, "x", 0, 2, 0.01);
50-
laminateInterval.add(CONFIG.colorStrata.laminateInterval!, "y", 0, 2, 0.01);
51-
const laminateDetail = colorStrata.addFolder("laminateDetail");
52-
laminateDetail.add(CONFIG.colorStrata.laminateDetail!, "x", 0, 10, 0.1);
53-
laminateDetail.add(CONFIG.colorStrata.laminateDetail!, "y", 0, 10, 0.1);
54-
const distortion = colorStrata.addFolder("distortion");
55-
distortion.add(CONFIG.colorStrata.distortion!, "x", 0, 10, 0.01);
56-
distortion.add(CONFIG.colorStrata.distortion!, "y", 0, 10, 0.01);
57-
const colorFactor = colorStrata.addFolder("colorFactor");
58-
colorFactor.add(CONFIG.colorStrata.colorFactor!, "x", 0, 10, 0.01);
59-
colorFactor.add(CONFIG.colorStrata.colorFactor!, "y", 0, 10, 0.01);
60-
colorFactor.add(CONFIG.colorStrata.colorFactor!, "z", 0, 10, 0.01);
61-
const timeStrength = colorStrata.addFolder("timeStrength");
62-
timeStrength.add(CONFIG.colorStrata.timeStrength!, "x", 0, 2, 0.01);
63-
timeStrength.add(CONFIG.colorStrata.timeStrength!, "y", 0, 2, 0.01);
64-
const noiseStrength = colorStrata.addFolder("noiseStrength");
65-
noiseStrength.add(CONFIG.colorStrata.noiseStrength!, "x", 0, 5, 0.01);
66-
noiseStrength.add(CONFIG.colorStrata.noiseStrength!, "y", 0, 5, 0.01);
67-
// color
68-
const color = gui.addFolder("color");
69-
color.addColor(CONFIG.color, "color0");
70-
color.addColor(CONFIG.color, "color1");
71-
color.addColor(CONFIG.color, "color2");
72-
color.addColor(CONFIG.color, "color3");
73-
};
74-
75-
const setConfig = () => {
76-
return {
77-
colorStrata: { ...CONFIG.colorStrata },
78-
color: { ...CONFIG.color },
79-
};
80-
};
26+
extend({ FxMaterial });
8127

8228
export const Playground = () => {
83-
const updateGUI = useGUI(setGUI);
29+
const ref = useRef<FxMaterialProps>();
8430

85-
const ref = useRef<THREE.ShaderMaterial>(null);
86-
const { size, dpr } = useThree((state) => {
87-
return { size: state.size, dpr: state.viewport.dpr };
88-
});
89-
90-
const [updateColorStrata] = useColorStrata({ size, dpr });
31+
const video = useVideoTexture("/gorilla.mov");
9132

9233
useFrame((props) => {
93-
const colorStrata = updateColorStrata(props, {
94-
...setConfig().colorStrata,
95-
texture: false,
96-
noise: false,
97-
});
98-
ref.current!.uniforms.u_fx.value = colorStrata;
99-
ref.current!.uniforms.u_color0.value = setConfig().color.color0;
100-
ref.current!.uniforms.u_color1.value = setConfig().color.color1;
101-
ref.current!.uniforms.u_color2.value = setConfig().color.color2;
102-
ref.current!.uniforms.u_color3.value = setConfig().color.color3;
103-
updateGUI();
34+
ref.current!.u_time = props.clock.getElapsedTime();
10435
});
10536

37+
// set resolution
38+
const { size } = useThree();
39+
useEffect(() => {
40+
ref.current!.u_resolution = new THREE.Vector2(size.width, size.height);
41+
}, [size]);
42+
10643
return (
10744
<mesh>
10845
<planeGeometry args={[2, 2]} />
109-
<shaderMaterial
46+
<fxMaterial
47+
key={FxMaterial.key}
48+
u_texture={video}
49+
u_keyColor={new THREE.Color(0x7bf43a)}
50+
u_textureResolution={new THREE.Vector2(1920, 1080)}
11051
ref={ref}
111-
vertexShader={`
112-
varying vec2 vUv;
113-
void main() {
114-
vUv = uv;
115-
gl_Position = vec4(position, 1.0);
116-
}
117-
`}
118-
fragmentShader={`
119-
precision highp float;
120-
varying vec2 vUv;
121-
uniform sampler2D u_fx;
122-
uniform vec3 u_color0;
123-
uniform vec3 u_color1;
124-
uniform vec3 u_color2;
125-
uniform vec3 u_color3;
126-
127-
void main() {
128-
vec2 uv = vUv;
129-
130-
vec2 map = texture2D(u_fx, uv).rg;
131-
vec2 normalizedMap = map * 2.0 - 1.0;
132-
133-
uv = uv * 2.0 - 1.0;
134-
uv *= mix(vec2(1.0), abs(normalizedMap), 3.0);
135-
uv = (uv + 1.0) / 2.0;
136-
137-
vec3 col = mix(mix(u_color0, u_color1, uv.x), mix(u_color2, u_color3, uv.x), uv.y);
138-
139-
gl_FragColor = vec4(col, 1.0);
140-
}
141-
`}
142-
uniforms={{
143-
u_fx: { value: null },
144-
u_color0: { value: null },
145-
u_color1: { value: null },
146-
u_color2: { value: null },
147-
u_color3: { value: null },
148-
}}
14952
/>
15053
</mesh>
15154
);

‎app/playground/main.frag‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
precision highp float;
2+
varying vec2 vUv;
3+
4+
uniform vec2 u_resolution;
5+
uniform vec2 u_textureResolution;
6+
uniform vec3 u_keyColor;
7+
8+
uniform sampler2D u_texture;
9+
10+
// From https://github.com/libretro/glsl-shaders/blob/master/nnedi3/shaders/rgb-to-yuv.glsl
11+
// 色の輝度（Y成分）に影響されずに、色の差異（彩度と色相）だけを使って背景を判定する
12+
vec2 RGBtoUV(vec3 rgb) {
13+
return vec2(
14+
rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5,
15+
rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5
16+
);
17+
}
18+
19+
void main() {
20+
// テクスチャー座標をアスペクト比に合わせる
21+
float screenAspect = u_resolution.x / u_resolution.y;
22+
float textureAspect = u_textureResolution.x / u_textureResolution.y;
23+
vec2 aspectRatio = vec2(
24+
min(screenAspect / textureAspect, 1.0),
25+
min(textureAspect / screenAspect, 1.0)
26+
);
27+
vec2 uv = vUv * aspectRatio + (1.0 - aspectRatio) * .5;
28+
29+
// texture の色を取得
30+
vec4 texColor = texture2D(u_texture, uv);
31+
32+
// マルチサンプリングと平均化処理
33+
float offset = 1.0 / u_textureResolution.y;
34+
vec4 sampleUp = texture2D(u_texture, uv + vec2(0.0, offset));
35+
vec4 sampleDown = texture2D(u_texture, uv + vec2(0.0, -offset));
36+
vec4 sampleLeft = texture2D(u_texture, uv + vec2(-offset, 0.0));
37+
vec4 sampleRight = texture2D(u_texture, uv + vec2(offset, 0.0));
38+
vec4 avgColor = (texColor + sampleUp + sampleDown + sampleLeft + sampleRight) / 5.0;
39+
40+
// key colorとの距離
41+
float chromaDist = distance(RGBtoUV(texColor.rgb), RGBtoUV(u_keyColor));
42+
43+
// 閾値
44+
float similarity = 0.2;
45+
float baseMask = chromaDist - similarity;
46+
47+
// 透明度の境界の滑らかさ
48+
float smoothness = 0.1;
49+
float fullMask = pow(clamp(baseMask / smoothness, 0., 1.), 1.5);
50+
texColor.a = fullMask;
51+
52+
// スピル抑制：クロマキーの反射部分の処理 値が大きいほど反射を削除する
53+
float spill = 1.1;
54+
float spillVal = pow(clamp(baseMask / spill, 0., 1.), 1.5);
55+
float desat = clamp(texColor.r * 0.2126 + texColor.g * 0.7152 + texColor.b * 0.0722, 0., 1.);
56+
texColor.rgb = mix(vec3(desat, desat, desat), texColor.rgb, spillVal);
57+
58+
59+
// ブレンディングによるアンチエイリアス処理の強化
60+
texColor = mix(texColor, avgColor, fullMask);
61+
62+
gl_FragColor = texColor;
63+
}
64+
65+
66+
67+
// ノイズリダクション
68+
// マルチサンプリングとブレンディングをして、境界のアンチエイリアス処理をしたい

‎app/playground/page.tsx‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,26 @@ import { Playground } from "./Playground";
33

44
export default function Page() {
55
return (
6-
<div style={{ width: "100%", height: "100svh", overflow: "hidden" }}>
7-
<ShaderFx>
8-
<Playground />
9-
</ShaderFx>
6+
<div style={{ width: "100%" }}>
7+
<div
8+
style={{
9+
position: "fixed",
10+
width: "100%",
11+
height: "100svh",
12+
pointerEvents: "none",
13+
}}>
14+
<ShaderFx>
15+
<Playground />
16+
</ShaderFx>
17+
</div>
18+
<a style={{ fontSize: "100px", fontWeight: "bold" }}>
19+
これはDOMですこれはDOMですこれはDOMですこれはDOMですこれはDOMです
20+
これはDOMです これはDOMです これはDOMです これはDOMです
21+
これはDOMです これはDOMです これはDOMです これはDOMです
22+
これはDOMです これはDOMです これはDOMです これはDOMです
23+
これはDOMです これはDOMです これはDOMです これはDOMです
24+
これはDOMです これはDOMです
25+
</a>
1026
</div>
1127
);
1228
}

0 commit comments

Comments
 (0)