WebGL Video

npx shadcn@latest add https://atelier-ui.com/r/webgl-video.json

npm install three @types/three @react-three/fiber

"use client"

import { useFrame } from "@react-three/fiber"
import { type ComponentRef, type RefObject, useLayoutEffect, useRef, useState } from "react"
import { type Mesh, SRGBColorSpace, type Texture, VideoTexture } from "three"
import { useDomPlane } from "../../hooks/use-dom-plane"
import { type Pointer, usePointerUv } from "../../hooks/use-pointer-uv"
import { applyUvCrop, computeObjectFit } from "../../lib/object-fit"
import { webglTeleport } from "../webgl-portal/webgl-portal"

export type { Pointer }

type WebglVideoProps = {
    src: string
    material?: (map: Texture, pointer: Pointer) => React.ReactNode
    webglEnabled?: boolean
    segments?: number
    zIndex?: number
    /**
     * Re-measures the DOM rect every frame so the plane follows animated parents (motion, parallax).
     * Costs one layout read per frame, so only enable it when needed.
     */
    autoReflow?: boolean
} & Omit<React.ComponentPropsWithoutRef<"video">, "children" | "src">

type PlaneProps = {
    el: RefObject<HTMLVideoElement | null>
    segments: number
    material?: (map: Texture, pointer: Pointer) => React.ReactNode
    pointer: Pointer
    uvFit: RefObject<{ x: number; y: number }>
    zIndex: number
    autoReflow: boolean
}

function Plane({ el, segments, material, pointer, uvFit, zIndex, autoReflow }: PlaneProps) {
    const mesh = useRef<Mesh>(null)
    const [texture, setTexture] = useState<VideoTexture | null>(null)
    const fitScale = useRef({ x: 1, y: 1 })
    const measureBounds = useDomPlane(el, mesh, { autoReflow, fitScale })

    useLayoutEffect(() => {
        const video = el.current
        if (!video) return

        /*
         * Build the texture from the DOM <video> itself so a single element
         * decodes once. VideoTexture pulls each new frame from it.
         */
        const videoTexture = new VideoTexture(video)
        videoTexture.colorSpace = SRGBColorSpace
        setTexture(videoTexture)
        return () => videoTexture.dispose()
    }, [el])

    useLayoutEffect(() => {
        const target = el.current
        if (!target || !texture) return

        const measure = () => {
            const m = mesh.current
            if (!m) return
            const rect = measureBounds()
            if (!rect) return

            /*
             * videoWidth/Height are 0 until metadata loads, which yields an
             * invalid aspect, so computeObjectFit skips cropping until then.
             */
            const video = texture.image as HTMLVideoElement
            const crop = computeObjectFit(
                rect.width / rect.height,
                video.videoWidth / video.videoHeight,
                getComputedStyle(target).objectFit,
            )

            fitScale.current.x = crop.fitScaleX
            fitScale.current.y = crop.fitScaleY
            pointer.repeat.set(crop.repeatU, crop.repeatV)
            uvFit.current.x = crop.repeatU / crop.fitScaleX
            uvFit.current.y = crop.repeatV / crop.fitScaleY

            applyUvCrop(m.geometry.attributes.uv, segments, crop.repeatU, crop.repeatV)
        }

        measure()

        /* Re-measure once the video reports its intrinsic size. */
        target.addEventListener("loadedmetadata", measure)
        target.addEventListener("resize", measure)

        const ro = new ResizeObserver(measure)
        ro.observe(target)
        ro.observe(document.body)
        return () => {
            ro.disconnect()
            target.removeEventListener("loadedmetadata", measure)
            target.removeEventListener("resize", measure)
        }
    }, [el, texture, segments, uvFit, pointer, measureBounds])

    useFrame(() => {
        /* Browsers without requestVideoFrameCallback need an explicit pull. */
        texture?.update()
    })

    if (!texture) return null

    return (
        <mesh ref={mesh} renderOrder={zIndex}>
            <planeGeometry args={[1, 1, segments, segments]} />
            {material ? (
                material(texture, pointer)
            ) : (
                <meshBasicMaterial map={texture} transparent />
            )}
        </mesh>
    )
}

export function WebglVideo({
    src,
    className,
    style,
    material,
    webglEnabled = true,
    segments = 1,
    zIndex = 0,
    autoReflow = false,
    autoPlay = true,
    muted = true,
    loop = true,
    playsInline = true,
    ...rest
}: WebglVideoProps) {
    const el = useRef<ComponentRef<"video">>(null)
    const uvFit = useRef({ x: 1, y: 1 })
    const pointer = usePointerUv(el, { enabled: webglEnabled, uvFit })

    return (
        <>
            <video
                ref={el}
                src={src}
                className={className}
                style={webglEnabled ? { ...style, opacity: 0 } : style}
                autoPlay={autoPlay}
                muted={muted}
                loop={loop}
                playsInline={playsInline}
                {...rest}
            />

            {webglEnabled && (
                <webglTeleport.In>
                    <Plane
                        el={el}
                        segments={segments}
                        material={material}
                        pointer={pointer}
                        uvFit={uvFit}
                        zIndex={zIndex}
                        autoReflow={autoReflow}
                    />
                </webglTeleport.In>
            )}
        </>
    )
}

import { type RefObject, useEffect, useMemo } from "react"
import { Vector2 } from "three"

export type Pointer = {
    uv: Vector2
    texUv: Vector2
    repeat: Vector2
    hover: number
}

type UsePointerUvOptions = {
    enabled: boolean
    /**
     * Maps element UVs into cropped texture UVs when object-fit trims the
     * media. Defaults to identity, so `texUv` mirrors `uv`.
     */
    uvFit?: RefObject<{ x: number; y: number }>
    getRect?: (el: HTMLElement) => DOMRect
}

/**
 * Tracks the cursor over a DOM element as normalized UVs, mutated in place so
 * shader materials can read it every frame without re-rendering React.
 */
export function usePointerUv(
    el: RefObject<HTMLElement | null>,
    { enabled, uvFit, getRect }: UsePointerUvOptions,
): Pointer {
    const pointer = useMemo<Pointer>(() => {
        return {
            uv: new Vector2(0.5, 0.5),
            texUv: new Vector2(0.5, 0.5),
            repeat: new Vector2(1, 1),
            hover: 0,
        }
    }, [])

    useEffect(() => {
        if (!enabled) return
        const target = el.current
        if (!target) return

        /*
         * Pointer events still fire on the DOM element through opacity:0,
         * so the browser tells us when the cursor is over it.
         */
        const onMove = (event: PointerEvent) => {
            const rect = getRect ? getRect(target) : target.getBoundingClientRect()
            const x = (event.clientX - rect.left) / rect.width
            const y = 1 - (event.clientY - rect.top) / rect.height
            const fit = uvFit?.current ?? { x: 1, y: 1 }
            pointer.uv.set(x, y)
            pointer.texUv.set(x * fit.x + (1 - fit.x) / 2, y * fit.y + (1 - fit.y) / 2)
        }

        const onEnter = () => (pointer.hover = 1)
        const onLeave = () => (pointer.hover = 0)

        target.addEventListener("pointermove", onMove)
        target.addEventListener("pointerenter", onEnter)
        target.addEventListener("pointerleave", onLeave)

        /*
         * Hover in too fast and pointerenter fires before these listeners
         * attach, so seed hover from the live :hover state instead.
         */
        if (target.matches(":hover")) pointer.hover = 1

        return () => {
            target.removeEventListener("pointermove", onMove)
            target.removeEventListener("pointerenter", onEnter)
            target.removeEventListener("pointerleave", onLeave)
        }
    }, [enabled, el, pointer, uvFit, getRect])

    return pointer
}

import { useFrame, useThree } from "@react-three/fiber"
import { type RefObject, useCallback, useRef } from "react"
import type { Mesh } from "three"

type UseDomPlaneOptions = {
    /**
     * Re-measures the DOM rect every frame so the plane follows animated
     * parents (motion, parallax). Costs one layout read per frame.
     */
    autoReflow: boolean
    fitScale?: RefObject<{ x: number; y: number }>
    getRect?: (el: HTMLElement) => DOMRect
}

/**
 * Positions and scales a mesh to cover a DOM element on the shared canvas.
 * Scroll is applied every frame. Layout changes are not observed here: the
 * caller calls `measureBounds` when the element resizes or repaints.
 */
export function useDomPlane(
    el: RefObject<HTMLElement | null>,
    mesh: RefObject<Mesh | null>,
    { autoReflow, fitScale, getRect }: UseDomPlaneOptions,
) {
    const size = useThree((state) => state.size)
    const viewport = useThree((state) => state.viewport)
    const bounds = useRef({ x: 0, y: 0, width: 0, height: 0 })

    const measureBounds = useCallback(() => {
        const target = el.current
        if (!target) return null

        /*
         * Rect in document coords so viewport position later needs only
         * window.scrollX/Y, instead of re-measuring bounds every render.
         */
        const rect = getRect ? getRect(target) : target.getBoundingClientRect()
        bounds.current.x = rect.left + window.scrollX
        bounds.current.y = rect.top + window.scrollY
        bounds.current.width = rect.width
        bounds.current.height = rect.height
        return rect
    }, [el, getRect])

    useFrame(() => {
        const m = mesh.current
        if (!m) return
        const pxToWorld = viewport.height / size.height
        const fit = fitScale?.current ?? { x: 1, y: 1 }

        const transitioning = document.documentElement.hasAttribute("data-atelier-transitioning")

        if ((autoReflow || transitioning) && el.current) {
            const rect = getRect ? getRect(el.current) : el.current.getBoundingClientRect()
            m.position.x = (rect.left + rect.width / 2 - size.width / 2) * pxToWorld
            m.position.y = -(rect.top + rect.height / 2 - size.height / 2) * pxToWorld
            m.scale.x = rect.width * pxToWorld * fit.x
            m.scale.y = rect.height * pxToWorld * fit.y
            return
        }

        const { x, y, width, height } = bounds.current
        m.position.x = (x + width / 2 - window.scrollX - size.width / 2) * pxToWorld
        m.position.y = -(y + height / 2 - window.scrollY - size.height / 2) * pxToWorld
        m.scale.x = width * pxToWorld * fit.x
        m.scale.y = height * pxToWorld * fit.y
    })

    return measureBounds
}

import type { BufferAttribute, InterleavedBufferAttribute } from "three"

export type ObjectFitCrop = {
    repeatU: number
    repeatV: number
    fitScaleX: number
    fitScaleY: number
}

/**
 * Replicates CSS object-fit on a WebGL plane: cover crops via UV repeat,
 * contain shrinks the mesh scale (UVs alone can't letterbox).
 *
 * Returns the neutral crop while the media aspect is unknown (e.g. a video
 * before its metadata loads reports 0x0).
 */
export function computeObjectFit(
    planeAspect: number,
    mediaAspect: number,
    objectFit: string,
): ObjectFitCrop {
    const crop: ObjectFitCrop = { repeatU: 1, repeatV: 1, fitScaleX: 1, fitScaleY: 1 }
    if (!Number.isFinite(mediaAspect) || mediaAspect <= 0) return crop

    if (objectFit === "cover") {
        if (planeAspect > mediaAspect) {
            crop.repeatV = mediaAspect / planeAspect
        } else {
            crop.repeatU = planeAspect / mediaAspect
        }
    } else if (objectFit === "contain") {
        if (planeAspect > mediaAspect) {
            crop.fitScaleX = mediaAspect / planeAspect
        } else {
            crop.fitScaleY = planeAspect / mediaAspect
        }
    }

    return crop
}

/** Rewrites a plane's UV grid so the texture samples the cropped region. */
export function applyUvCrop(
    uvAttribute: BufferAttribute | InterleavedBufferAttribute,
    segments: number,
    repeatU: number,
    repeatV: number,
) {
    const offsetU = (1 - repeatU) / 2
    const offsetV = (1 - repeatV) / 2

    for (let iy = 0; iy <= segments; iy++) {
        for (let ix = 0; ix <= segments; ix++) {
            const index = iy * (segments + 1) + ix
            const u = ix / segments
            const v = 1 - iy / segments
            uvAttribute.setXY(index, u * repeatU + offsetU, v * repeatV + offsetV)
        }
    }

    uvAttribute.needsUpdate = true
}

"use client"

import {
    type ReactNode,
    Suspense,
    useEffect,
    useId,
    useLayoutEffect,
    useSyncExternalStore,
} from "react"

const useIsoLayoutEffect = typeof window !== "undefined" ? useLayoutEffect : useEffect

// Minimal teleport: <In> registers children in an external store,
// <Out> renders them — bridges across the Canvas React root the same
function WebglTeleport() {
    const items = new Map<string, ReactNode>()
    const listeners = new Set<() => void>()
    let snapshot: [string, ReactNode][] = []

    const emit = () => {
        snapshot = Array.from(items.entries())
        for (const listener of listeners) {
            listener()
        }
    }

    const subscribe = (listener: () => void) => {
        listeners.add(listener)
        return () => {
            listeners.delete(listener)
        }
    }
    const getSnapshot = () => snapshot

    function useItems() {
        return useSyncExternalStore(subscribe, getSnapshot, getSnapshot)
    }

    return {
        In({ children }: { children: ReactNode }) {
            const id = useId()

            useIsoLayoutEffect(() => {
                items.set(id, children)
                emit()
                return () => {
                    items.delete(id)
                    emit()
                }
            }, [id, children])
            return null
        },
        useItems,
        Out() {
            const list = useItems()
            return (
                <>
                    {list.map(([id, node]) => (
                        <Suspense key={id} fallback={null}>
                            {node}
                        </Suspense>
                    ))}
                </>
            )
        },
    }
}

const webglTeleport = WebglTeleport()
const effectTeleport = WebglTeleport()

export function WebglPortal() {
    return <webglTeleport.Out />
}

export { effectTeleport, webglTeleport }

"use client"

import { advance, Canvas, type CanvasProps, useStore, useThree } from "@react-three/fiber"
import { EffectComposer } from "@react-three/postprocessing"
import { cancelFrame, type FrameData, frame } from "motion"
import { type ComponentRef, type ReactNode, useEffect, useRef, useState } from "react"
import type { Camera, Scene } from "three"
import { effectTeleport, WebglPortal } from "../webgl-portal/webgl-portal"

type WebglProviderProps = Omit<CanvasProps, "children" | "eventSource"> & {
    children: ReactNode
    className?: string
    contained?: boolean
}

type WebglReadyOptions = {
    scene?: Scene
    camera?: Camera
    enabled?: boolean
    onReady?: () => void
}

export function useWebglReady({ scene, camera, enabled = true, onReady }: WebglReadyOptions = {}) {
    const [ready, setReady] = useState(false)
    const gl = useThree((state) => state.gl)
    const defaultScene = useThree((state) => state.scene)
    const defaultCamera = useThree((state) => state.camera)
    const onReadyRef = useRef(onReady)
    onReadyRef.current = onReady

    const targetScene = scene ?? defaultScene
    const targetCamera = camera ?? defaultCamera

    useEffect(() => {
        if (!enabled) return
        let active = true

        gl.compileAsync(targetScene, targetCamera).then(() => {
            if (!active) return
            requestAnimationFrame(() => {
                if (!active) return
                setReady(true)
                onReadyRef.current?.()
            })
        })

        return () => {
            active = false
        }
    }, [gl, targetScene, targetCamera, enabled])

    return ready
}

// Renders in Motion's `postRender` phase, after Lenis and Motion have
// updated. One shared driver serves every mounted provider.
type CanvasStore = ReturnType<typeof useStore>
const canvasStores = new Set<CanvasStore>()
let clockStart: number | null = null

function tick(data: FrameData) {
    if (clockStart === null) clockStart = data.timestamp

    // frameloop="never" expects the elapsed clock time in seconds.
    const elapsed = (data.timestamp - clockStart) / 1000

    let runGlobalEffects = true
    for (const store of canvasStores) {
        const state = store.getState()
        if (state.internal.active) {
            advance(elapsed, runGlobalEffects, state)
            runGlobalEffects = false
        }
    }
}

function MotionFrameloop() {
    const store = useStore()

    useEffect(() => {
        canvasStores.add(store)
        if (canvasStores.size === 1) frame.postRender(tick, true)
        return () => {
            canvasStores.delete(store)
            if (canvasStores.size === 0) cancelFrame(tick)
        }
    }, [store])

    return null
}

function Effects() {
    const effects = effectTeleport.useItems()
    const gl = useThree((state) => state.gl)
    const mounted = effects.length > 0

    // EffectComposer sets `renderer.autoClear = false` and never restores it;
    // without this the canvas keeps its last frame once the composer unmounts.
    useEffect(() => {
        if (!mounted) return
        return () => {
            gl.autoClear = true
        }
    }, [mounted, gl])

    if (!mounted) return null

    return (
        <EffectComposer key={effects.length}>
            <effectTeleport.Out />
        </EffectComposer>
    )
}

export function WebglProvider({
    children,
    className,
    style,
    contained = false,
    ...canvasProps
}: WebglProviderProps) {
    const [eventSource, setEventSource] = useState<ComponentRef<"div"> | null>(null)

    return (
        <div
            ref={setEventSource}
            data-atelier-webgl=""
            className={className}
            style={contained ? { position: "relative" } : { display: "contents" }}
        >
            <Canvas
                eventPrefix="client"
                dpr={[1, 2]}
                {...canvasProps}
                frameloop="never"
                eventSource={eventSource ?? undefined}
                style={{
                    position: contained ? "absolute" : "fixed",
                    inset: 0,
                    pointerEvents: "none",
                    ...style,
                }}
            >
                <MotionFrameloop />
                <WebglPortal />
                <Effects />
            </Canvas>

            {children}
        </div>
    )
}

import { WebglProvider } from "@/components/webgl-provider/webgl-provider";

export default function RootLayout({ children }) {
  return <WebglProvider>{children}</WebglProvider>;
}

<WebglVideo src="/clip.mp4" className="w-full h-auto" />

<WebglVideo src="/clip.mp4" className="w-full h-64 object-cover" />

<WebglVideo
  src="/clip.mp4"
  material={(map, pointer) => (
    <myShaderMaterial
      uMap={map}
      uPointer={pointer.uv}
      uHover={pointer.hover}
      transparent
    />
  )}
/>