babylonjs 实现显示实时安防网络摄像头画面的材质

一、参考连接

官方资料：https://doc.babylonjs.com/features/featuresDeepDive/materials/using/videoTexture
播放案例：
https://playground.babylonjs.com/#ZMCFYA#83
https://playground.babylonjs.com/#1BYH8W#4
本地摄像头：https://www.babylonjs-playground.com/#1R77YT#32
video标签播放：https://www.babylonjs-playground.com/#9FSDC7#13
video标签：https://www.babylonjs-playground.com/#9FSDC7#271
hsl源：https://playground.babylonjs.com/#GEA3P0
hsl源：https://playground.babylonjs.com/#YJLYCB#10
dashjs源+video标签：https://playground.babylonjs.com/#9FSDC7#234

二、资料总结

babylonjs 可以播放mp4等视频文件；
babylonjs 支持通过webrtc播放网络摄像头视频；
babylonjs 支持通过获取本机摄像头实时视频；
babylonjs 内部创建了一个HTML Video标签，当video标签被dispose的时候，视频会中断；

三、动手实践

经过初步研究后发现可能无法直接播放zlm的webrtc，可能只能选择通过隐藏的video标签播放。

以下代码后端的源使用了zlmediakit流媒体服务器。

左侧是原始摄像头流、中间是canvas镜像流，右侧是babylons材质的效果：

单独展示babylonjs的效果：

多边形UV映射：

代码见下：

<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8">
    <title>Title</title>
    <script src="./jessibuca.js"></script>
    <script src="https://cdn.babylonjs.com/babylon.js"></script>
    <style>
        #wrapper {
            display: flex;
        }

        #container {
            /* 你可以根据需要设置宽度 */
            height: 500px;
            width: 33%;
            background-color: lightblue;
            /*display: none;*/
        }
        #mirror{
            height: 500px;
            width:33%;
            background-color: greenyellow;
            /*display: none;*/
        }
        #renderCanvasDiv {
            /* 你可以根据需要设置宽度 */
            height: 500px;
            width: 33%;
            background-color: lightcoral;
            /* 示例背景色 */
        }


        #renderCanvas {
            width: 100%;
            height: 100%;
            touch-action: none;
        }
    </style>
</head>

<body>
    <div id="wrapper">
        <div id="container"></div>
        <div id="mirror"><canvas id="mirrorCanvas"></canvas></div>
        <div id="renderCanvasDiv"><canvas id="renderCanvas"></canvas></div>

    </div>

    <script>
        var $container = document.getElementById('container');

        var showOperateBtns = false; // 是否显示按钮
        var forceNoOffscreen = true; //
        var jessibuca = null;

        function createStream() {
            jessibuca = null;
            jessibuca = new Jessibuca({
                container: $container,
                videoBuffer: 0.2, // 缓存时长
                isResize: false,
                text: "",
                loadingText: "",
                useMSE: false,
                debug: false,
                showBandwidth: showOperateBtns, // 显示网速
                operateBtns: {
                    fullscreen: showOperateBtns,
                    screenshot: showOperateBtns,
                    play: showOperateBtns,
                    audio: false,
                    recorder: false
                },
                forceNoOffscreen: forceNoOffscreen,
                isNotMute: false,
            },);
            var href = "wss://hd.iios.com.cn/zlmediakit/IIOS-CONSOLE/241113LH.live.flv?vhost=__defaultVhost__&session=BFF1E27336359CBA61C967E1D8F1AAAB";
            jessibuca.play(href);
            createBbl();
        }


        function refreshMirror(){

            const sourceCanvas = document.querySelector('#container canvas');
            const newCanvas = document.getElementById("mirrorCanvas");
            const newCtx = newCanvas.getContext('2d');

            newCanvas.width = sourceCanvas.width;
            newCanvas.height = sourceCanvas.height;

            newCtx.clearRect(0, 0, newCanvas.width, newCanvas.height);
            newCtx.drawImage(sourceCanvas, 0, 0, newCanvas.width, newCanvas.height);
        }


        function createBbl() {
            const sourceCanvas = document.querySelector('#container canvas');
            if (sourceCanvas) {
                console.log("sourceCanvas", sourceCanvas)
                const canvas = document.getElementById("renderCanvas");
                const engine = new BABYLON.Engine(canvas, true);
                const scene = new BABYLON.Scene(engine);
                var camera = new BABYLON.ArcRotateCamera("Camera", Math.PI / 2, Math.PI / 2, 2, BABYLON.Vector3.Zero(), scene);
                camera.attachControl(canvas, true);
                // 创建一个光源
                // 创建朝上的光源
                const light1 = new BABYLON.HemisphericLight("light1", BABYLON.Vector3.Up(), scene);
                light1.intensity = 0.7; // 设置光强度

                // 创建朝下的光源
                const light2 = new BABYLON.HemisphericLight("light2", BABYLON.Vector3.Down(), scene);
                light2.intensity = 0.7; // 设置光强度

                // 创建平面
                var plane = BABYLON.MeshBuilder.CreateGround(
                    "plane",  // 平面名称
                    { width: 10, height: 10 }, // 设置平面的宽度和高度
                    scene
                );

                // 创建球体
                var sphere = BABYLON.MeshBuilder.CreateSphere(
                    "sphere",  // 球体名称
                    { diameter: 2 }, // 设置球体的直径
                    scene
                );
                sphere.position.y = 1; // 设置球体的Y轴位置，让它悬浮在平面上方
                // 创建半立方体
                const halfCube = BABYLON.MeshBuilder.CreateBox("halfCube", { width: 2, height: 1, depth: 2 }, scene);
                halfCube.position.y = 3;
                const dynamicTexture = new BABYLON.DynamicTexture(
                    "dynamicTexture",
                    { width: 640, height: 360 },
                    scene,
                    false
                );

                // 创建多边形
                
                const positions = [];
                const indices = [];
                const uvs = [];
                const radius = 10;
                const sides = 9;

                // 计算顶点位置（在XZ平面上）
                // 计算顶点位置（在XZ平面上）
                for (let i = 0; i < sides; i++) {
                    const angle = (2 * Math.PI / sides) * i;
                    positions.push(Math.cos(angle) * radius, 0, Math.sin(angle) * radius);
                    uvs.push((Math.cos(angle) + 1) / 2, (Math.sin(angle) + 1) / 2); // UV坐标
                }


                // 中心顶点（在XZ平面上）
                positions.push(0, 0, 0);
                uvs.push(0.5, 0.5);

                // 计算索引
                for (let i = 0; i < sides; i++) {
                    indices.push(i, (i + 1) % sides, sides);
                }

                // 创建自定义网格
                const customMesh = new BABYLON.Mesh("custom", scene);
                const vertexData = new BABYLON.VertexData();

                vertexData.positions = positions;
                vertexData.indices = indices;
                vertexData.uvs = uvs;
                vertexData.applyToMesh(customMesh);
                customMesh.position.y = -3

                var material = new BABYLON.StandardMaterial("material", scene);
                material.diffuseTexture = dynamicTexture;
                material.reflectionTexture = null;
                material.useReflection = false;  // 禁用反射
                material.useMicroSurfaceFromReflectivityMap = false;  // 禁用反射微表面效果
                material.environmentTexture = null;  // 禁用环境光反射
                material.specularColor = new BABYLON.Color3(0, 0, 0); // 无高光反射
                material.environmentIntensity = 0; // 禁用环境光反射

                sphere.material = material;
                plane.material = material;
                halfCube.material = material;
                customMesh.material = material;
                customMesh.material.backFaceCulling = false;

                function updateTexture() {
                    if (!sourceCanvas) {
                        console.error("sourceCanvas is not found or not initialized.");
                        return;
                    }

                    const ctx = dynamicTexture.getContext();
                    if (!ctx) {
                        console.error("Failed to get context from dynamicTexture.");
                        return;
                    }

                    // Clear the dynamic texture
                    ctx.clearRect(0, 0, dynamicTexture.getSize().width, dynamicTexture.getSize().height);

                    // Draw the source canvas onto the dynamic texture
                    ctx.drawImage(sourceCanvas, 0, 0, dynamicTexture.getSize().width, dynamicTexture.getSize().height);

                    // Update the dynamic texture to apply changes
                    dynamicTexture.update();

                    console.log("update", sourceCanvas.width, sourceCanvas.height);
                }

                engine.runRenderLoop(function () {
                    refreshMirror();
                    updateTexture();
                    scene.render();
                });

                window.addEventListener("resize", function () {
                    engine.resize();
                });

            }

        }


        createStream();


    </script>

</body>

</html>

完整源代码：claves-babylonjs-camera 下载