Skip to content

Commit ed9194d

Browse files
author
takuma-hmng8
committed
add useChromaKey
1 parent abd2886 commit ed9194d

File tree

10 files changed

+287
-87
lines changed

10 files changed

+287
-87
lines changed

‎app/ShaderFx.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ export const ShaderFx = ({ children }: { children: React.ReactNode }) => {
1313
factor={1}
1414
onChange={({ factor }) => {
1515
console.log(`dpr:${dpr}`);
16-
setDpr(Math.round((0.5 + 1.0 * factor) * 10) / 10);
16+
// setDpr(Math.round((0.5 + 1.0 * factor) * 10) / 10);
1717
}}>
1818
<Suspense fallback={null}>{children}</Suspense>
1919
<Perf position={"bottom-left"} minimal={false} />

‎app/playground/FxMaterial.tsx‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,14 @@ declare global {
1313
export type FxMaterialProps = {
1414
u_time: number;
1515
u_resolution: THREE.Vector2;
16-
u_textureResolution: THREE.Vector2;
17-
u_texture: THREE.Texture;
18-
u_keyColor: THREE.Color;
16+
u_fx: THREE.Texture;
1917
};
2018

2119
export const FxMaterial = shaderMaterial(
2220
{
2321
u_time: 0,
2422
u_resolution: new THREE.Vector2(0, 0),
25-
u_textureResolution: new THREE.Vector2(0, 0),
26-
u_texture: null,
27-
u_keyColor: new THREE.Color(0x00ff00),
23+
u_fx: new THREE.Texture(),
2824
},
2925

3026
`

‎app/playground/Playground.tsx‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
useFxBlending,
1010
useColorStrata,
1111
useBrightnessPicker,
12+
useChromaKey,
1213
} from "@/packages/use-shader-fx/src";
1314
import {
1415
NoiseParams,
@@ -21,35 +22,43 @@ import {
2122
import GUI from "lil-gui";
2223
import { useGUI } from "@/utils/useGUI";
2324
import { FxMaterial, FxMaterialProps } from "./FxMaterial";
24-
import { useVideoTexture } from "@react-three/drei";
25+
import { useTexture, useVideoTexture } from "@react-three/drei";
2526

2627
extend({ FxMaterial });
2728

2829
export const Playground = () => {
2930
const ref = useRef<FxMaterialProps>();
3031

31-
const video = useVideoTexture("/gorilla.mov");
32+
const video = useVideoTexture("/tv.mov");
33+
const sample = useTexture("/test.jpg");
3234

33-
useFrame((props) => {
34-
ref.current!.u_time = props.clock.getElapsedTime();
35+
const { size, viewport } = useThree();
36+
37+
const [updateChromakey, set, { output }] = useChromaKey({
38+
size,
39+
dpr: viewport.dpr,
40+
});
41+
42+
set({
43+
texture: video,
44+
textureResolution: new THREE.Vector2(1920, 1080),
45+
keyColor: new THREE.Color(0x00ff00),
46+
similarity: 0.5,
47+
spill: 0.2,
48+
smoothness: 0.0,
49+
contrast: 1.8,
50+
gamma: 0.7,
51+
brightness: 0.0,
3552
});
3653

37-
// set resolution
38-
const { size } = useThree();
39-
useEffect(() => {
40-
ref.current!.u_resolution = new THREE.Vector2(size.width, size.height);
41-
}, [size]);
54+
useFrame((props) => {
55+
updateChromakey(props);
56+
});
4257

4358
return (
4459
<mesh>
4560
<planeGeometry args={[2, 2]} />
46-
<fxMaterial
47-
key={FxMaterial.key}
48-
u_texture={video}
49-
u_keyColor={new THREE.Color(0x7bf43a)}
50-
u_textureResolution={new THREE.Vector2(1920, 1080)}
51-
ref={ref}
52-
/>
61+
<fxMaterial key={FxMaterial.key} u_fx={output} ref={ref} />
5362
</mesh>
5463
);
5564
};

‎app/playground/main.frag‎

Lines changed: 3 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,68 +1,7 @@
11
precision highp float;
22
varying vec2 vUv;
3-
4-
uniform vec2 u_resolution;
5-
uniform vec2 u_textureResolution;
6-
uniform vec3 u_keyColor;
7-
8-
uniform sampler2D u_texture;
9-
10-
// From https://github.com/libretro/glsl-shaders/blob/master/nnedi3/shaders/rgb-to-yuv.glsl
11-
// 色の輝度（Y成分）に影響されずに、色の差異（彩度と色相）だけを使って背景を判定する
12-
vec2 RGBtoUV(vec3 rgb) {
13-
return vec2(
14-
rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5,
15-
rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5
16-
);
17-
}
3+
uniform sampler2D u_fx;
184

195
void main() {
20-
// テクスチャー座標をアスペクト比に合わせる
21-
float screenAspect = u_resolution.x / u_resolution.y;
22-
float textureAspect = u_textureResolution.x / u_textureResolution.y;
23-
vec2 aspectRatio = vec2(
24-
min(screenAspect / textureAspect, 1.0),
25-
min(textureAspect / screenAspect, 1.0)
26-
);
27-
vec2 uv = vUv * aspectRatio + (1.0 - aspectRatio) * .5;
28-
29-
// texture の色を取得
30-
vec4 texColor = texture2D(u_texture, uv);
31-
32-
// マルチサンプリングと平均化処理
33-
float offset = 1.0 / u_textureResolution.y;
34-
vec4 sampleUp = texture2D(u_texture, uv + vec2(0.0, offset));
35-
vec4 sampleDown = texture2D(u_texture, uv + vec2(0.0, -offset));
36-
vec4 sampleLeft = texture2D(u_texture, uv + vec2(-offset, 0.0));
37-
vec4 sampleRight = texture2D(u_texture, uv + vec2(offset, 0.0));
38-
vec4 avgColor = (texColor + sampleUp + sampleDown + sampleLeft + sampleRight) / 5.0;
39-
40-
// key colorとの距離
41-
float chromaDist = distance(RGBtoUV(texColor.rgb), RGBtoUV(u_keyColor));
42-
43-
// 閾値
44-
float similarity = 0.2;
45-
float baseMask = chromaDist - similarity;
46-
47-
// 透明度の境界の滑らかさ
48-
float smoothness = 0.1;
49-
float fullMask = pow(clamp(baseMask / smoothness, 0., 1.), 1.5);
50-
texColor.a = fullMask;
51-
52-
// スピル抑制：クロマキーの反射部分の処理 値が大きいほど反射を削除する
53-
float spill = 1.1;
54-
float spillVal = pow(clamp(baseMask / spill, 0., 1.), 1.5);
55-
float desat = clamp(texColor.r * 0.2126 + texColor.g * 0.7152 + texColor.b * 0.0722, 0., 1.);
56-
texColor.rgb = mix(vec3(desat, desat, desat), texColor.rgb, spillVal);
57-
58-
59-
// ブレンディングによるアンチエイリアス処理の強化
60-
texColor = mix(texColor, avgColor, fullMask);
61-
62-
gl_FragColor = texColor;
63-
}
64-
65-
66-
67-
// ノイズリダクション
68-
// マルチサンプリングとブレンディングをして、境界のアンチエイリアス処理をしたい
6+
gl_FragColor = texture2D(u_fx, vUv);
7+
}

‎packages/use-shader-fx/src/hooks/useChromaKey/index.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { useCallback, useMemo } from "react";
2+
import * as THREE from "three";
3+
import { useMesh } from "./useMesh";
4+
import { useCamera } from "../../utils/useCamera";
5+
import { RootState } from "@react-three/fiber";
6+
import { useSingleFBO } from "../../utils/useSingleFBO";
7+
import { setUniform } from "../../utils/setUniforms";
8+
import { HooksProps, HooksReturn } from "../types";
9+
import { useParams } from "../../utils/useParams";
10+
11+
export type ChromaKeyParams = {
12+
/** Process this texture with chroma key , default:THREE.Texture */
13+
texture?: THREE.Texture;
14+
/** texture resolution , default:THREE.Vector2(0, 0) */
15+
textureResolution?: THREE.Vector2;
16+
/** key color for chromakey processing , default: THREE.Color(0x00ff00) */
17+
keyColor?: THREE.Color;
18+
/** If the similarity with the key color exceeds this value, it becomes transparent. , default: 0.01 */
19+
similarity?: number;
20+
/** smoothness , default : 0.01 */
21+
smoothness?: number;
22+
/** spill , default : 0.01 */
23+
spill?: number;
24+
/** tone correction , default : THREE.Vector4(1.0, 1.0, 1.0, 1.0) */
25+
color?: THREE.Vector4;
26+
/** contrast , default : 1.0 */
27+
contrast?: number;
28+
/** brightness , default : 0.0 */
29+
brightness?: number;
30+
/** gamma correction , default : 1.0 */
31+
gamma?: number;
32+
};
33+
34+
export type ChromaKeyObject = {
35+
scene: THREE.Scene;
36+
material: THREE.Material;
37+
camera: THREE.Camera;
38+
renderTarget: THREE.WebGLRenderTarget;
39+
output: THREE.Texture;
40+
};
41+
42+
export const CHROMAKEY_PARAMS: ChromaKeyParams = {
43+
texture: new THREE.Texture(),
44+
textureResolution: new THREE.Vector2(0, 0),
45+
keyColor: new THREE.Color(0x00ff00),
46+
similarity: 0.01,
47+
smoothness: 0.01,
48+
spill: 0.01,
49+
color: new THREE.Vector4(1.0, 1.0, 1.0, 1.0),
50+
contrast: 1.0,
51+
brightness: 0.0,
52+
gamma: 1.0,
53+
};
54+
55+
/**
56+
* @link https://github.com/takuma-hmng8/use-shader-fx#usage
57+
*/
58+
export const useChromaKey = ({
59+
size,
60+
dpr,
61+
samples = 0,
62+
}: HooksProps): HooksReturn<ChromaKeyParams, ChromaKeyObject> => {
63+
const scene = useMemo(() => new THREE.Scene(), []);
64+
const material = useMesh({ scene, size, dpr });
65+
const camera = useCamera(size);
66+
const [renderTarget, updateRenderTarget] = useSingleFBO({
67+
scene,
68+
camera,
69+
size,
70+
dpr,
71+
samples,
72+
});
73+
74+
const [params, setParams] = useParams<ChromaKeyParams>(CHROMAKEY_PARAMS);
75+
76+
const updateFx = useCallback(
77+
(props: RootState, updateParams?: ChromaKeyParams) => {
78+
const { gl } = props;
79+
updateParams && setParams(updateParams);
80+
81+
setUniform(material, "u_texture", params.texture!);
82+
setUniform(material, "u_textureResolution", params.textureResolution!);
83+
setUniform(material, "u_keyColor", params.keyColor!);
84+
setUniform(material, "u_similarity", params.similarity!);
85+
setUniform(material, "u_smoothness", params.smoothness!);
86+
setUniform(material, "u_spill", params.spill!);
87+
setUniform(material, "u_color", params.color!);
88+
setUniform(material, "u_contrast", params.contrast!);
89+
setUniform(material, "u_brightness", params.brightness!);
90+
setUniform(material, "u_gamma", params.gamma!);
91+
92+
return updateRenderTarget(gl);
93+
},
94+
[updateRenderTarget, material, setParams, params]
95+
);
96+
97+
return [
98+
updateFx,
99+
setParams,
100+
{
101+
scene: scene,
102+
material: material,
103+
camera: camera,
104+
renderTarget: renderTarget,
105+
output: renderTarget.texture,
106+
},
107+
];
108+
};

‎packages/use-shader-fx/src/hooks/useChromaKey/shader/main.frag‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
precision highp float;
2+
varying vec2 vUv;
3+
4+
uniform sampler2D u_texture;
5+
uniform vec2 u_resolution;
6+
uniform vec2 u_textureResolution;
7+
8+
uniform vec3 u_keyColor;
9+
uniform float u_similarity;
10+
uniform float u_smoothness;
11+
uniform float u_spill;
12+
13+
uniform vec4 u_color;
14+
uniform float u_contrast;
15+
uniform float u_brightness;
16+
uniform float u_gamma;
17+
18+
// From https://github.com/libretro/glsl-shaders/blob/master/nnedi3/shaders/rgb-to-yuv.glsl
19+
vec2 RGBtoUV(vec3 rgb) {
20+
return vec2(
21+
rgb.r * -0.169 + rgb.g * -0.331 + rgb.b * 0.5 + 0.5,
22+
rgb.r * 0.5 + rgb.g * -0.419 + rgb.b * -0.081 + 0.5
23+
);
24+
}
25+
float getChromeDist(vec3 texColor){
26+
float chromaDist = distance(RGBtoUV(texColor), RGBtoUV(u_keyColor));
27+
return chromaDist;
28+
}
29+
30+
float getBoxFilteredChromaDist(vec3 rgb, vec2 uv)
31+
{
32+
vec2 pixel_size = vec2(1.,1.);
33+
vec2 h_pixel_size = pixel_size / 2.0;
34+
vec2 point_0 = vec2(pixel_size.x, h_pixel_size.y);
35+
vec2 point_1 = vec2(h_pixel_size.x, -pixel_size.y);
36+
float distVal = getChromeDist(texture2D(u_texture,uv-point_0).rgb);
37+
distVal += getChromeDist(texture2D(u_texture,uv+point_0).rgb);
38+
distVal += getChromeDist(texture2D(u_texture,uv-point_1).rgb);
39+
distVal += getChromeDist(texture2D(u_texture,uv+point_1).rgb);
40+
distVal *= 2.0;
41+
distVal += getChromeDist(rgb);
42+
return distVal / 9.0;
43+
}
44+
45+
vec4 CalcColor(vec4 rgba)
46+
{
47+
return vec4(pow(rgba.rgb, vec3(u_gamma, u_gamma, u_gamma)) * u_contrast + u_brightness, rgba.a);
48+
}
49+
50+
void main() {
51+
float screenAspect = u_resolution.x / u_resolution.y;
52+
float textureAspect = u_textureResolution.x / u_textureResolution.y;
53+
vec2 aspectRatio = vec2(
54+
min(screenAspect / textureAspect, 1.0),
55+
min(textureAspect / screenAspect, 1.0)
56+
);
57+
vec2 uv = vUv * aspectRatio + (1.0 - aspectRatio) * .5;
58+
59+
vec4 texColor = texture2D(u_texture, uv);
60+
texColor.rgb *= (texColor.a > 0.) ? (1. / texColor.a) : 0.;
61+
62+
float chromaDist = getBoxFilteredChromaDist(texColor.rgb,uv);
63+
64+
float baseMask = chromaDist - u_similarity;
65+
float fullMask = pow(clamp(baseMask / u_smoothness, 0., 1.), 1.5);
66+
67+
texColor.rgba *= u_color;
68+
texColor.a = fullMask;
69+
70+
float spillVal = pow(clamp(baseMask / u_spill, 0., 1.), 1.5);
71+
float desat = clamp(texColor.r * 0.2126 + texColor.g * 0.7152 + texColor.b * 0.0722, 0., 1.);
72+
texColor.rgb = mix(vec3(desat, desat, desat), texColor.rgb, spillVal);
73+
74+
vec4 finColor = CalcColor(texColor);
75+
76+
gl_FragColor = finColor;
77+
}

‎packages/use-shader-fx/src/hooks/useChromaKey/shader/main.vert‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
varying vec2 vUv;
2+
3+
void main() {
4+
vUv = uv;
5+
gl_Position = vec4(position, 1.0);
6+
}

0 commit comments

Comments
 (0)