diff --git a/sdk/package.json b/sdk/package.json index c7907a042b587f4e063342559f7ab2245fb0797a..cda0ecbacf05e14263691caa9c16bbb9d8795948 100644 --- a/sdk/package.json +++ b/sdk/package.json @@ -17,6 +17,7 @@ ], "dependencies": { "core-js": "^2.6.11", + "gl-matrix": "^3.4.3", "jmuxer": "1.2.0", "webworkify-webpack": "2.1.5" }, diff --git a/sdk/src/CameraPlayer.js b/sdk/src/CameraPlayer.js index 87db178b31e307734adfda758a65c0433b1fef3a..19846d88d0522815a850b928940c199efe035572 100644 --- a/sdk/src/CameraPlayer.js +++ b/sdk/src/CameraPlayer.js @@ -13,6 +13,7 @@ export default class CameraPlayer { if (this.options.videoOptions) { this.video = document.createElement('video'); this.canvas = document.createElement('canvas'); + this.initWebGl(); this.videoStream = null; this.animationId = null; this.stopVideo = false; @@ -57,22 +58,12 @@ export default class CameraPlayer { } render() { - const ctx = this.canvas.getContext('2d', {willReadFrequently: true}); - ctx.clearRect(0, 0, this.canvas.width,this.canvas.height); - ctx.save; - if (this.options.devId === CAMERA_COMMON_MODE_MAP.USER) { - // 前置摄像头 - ctx.rotate(Math.PI/2); - ctx.drawImage(this.video, 0, -this.canvas.width, this.canvas.height, this.canvas.width); - - } else { - // 后置摄像头 - ctx.rotate(-90 * Math.PI / 180); - ctx.drawImage(this.video, -this.canvas.height, 0, this.canvas.height, this.canvas.width); - } - ctx.restore(); - - const imageData = ctx.getImageData(0, 0, this.canvas.width, this.canvas.height); + this.updateTexture(); + // 创建一个Uint8Array数组，用于存储像素数据 + const pixels = new Uint8Array(this.canvas.width * this.canvas.height * 4); + // 读取像素数据 + this.gl.readPixels(0, 0, this.canvas.width, this.canvas.height, this.gl.RGBA, this.gl.UNSIGNED_BYTE, pixels); + const imageData = this.getImageData(pixels, this.canvas.width, this.canvas.height); this.videoEncoderWorker.postMessage({ type: 'encode', @@ -80,6 +71,14 @@ export default class CameraPlayer { }); } + getImageData(data, width, height) { + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + const imageData = ctx.createImageData(width, height); + imageData.data.set(data); + return imageData; + } + videoStreamHandle() { const newVideoStream = new MediaStream(); newVideoStream.addTrack(this.videoStream); @@ -93,6 +92,16 @@ export default class CameraPlayer { height: this.canvas.height, x264WasmPath: this.options.x264WasmPath }); + + // 指定宽高 + this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); + // 绑定纹理对象 + this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture); + // 将视频帧绘制到纹理对象 + this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, this.video); + // 绘制纹理 + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + }); this.video.play(); @@ -117,6 +126,14 @@ export default class CameraPlayer { } break; } + case 'setRotateData': { + const rotateData = message.data; + this.gl.uniformMatrix4fv(this.rotationMatrixLocation, false, rotateData); + this.gl.uniform1i(this.textureLocation, 0); + // 绘制纹理 + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + break; + } default: break; } @@ -130,4 +147,105 @@ export default class CameraPlayer { this.videoStream.stop(); } } + + initWebGl() { + // 获取webgl上下文 + this.gl = this.canvas.getContext('webgl'); + // 创建顶点着色器 + const vertexShaderSource = ` + attribute vec4 a_position; + attribute vec2 a_texCoord; + uniform mat4 u_rotationMatrix; + varying vec2 v_texCoord; + void main() { + gl_Position = u_rotationMatrix * a_position; + v_texCoord = a_texCoord; + }`; + const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER); + this.gl.shaderSource(vertexShader, vertexShaderSource); + this.gl.compileShader(vertexShader); + + // 创建片元着色器 + const fragmentShaderSource = ` + precision mediump float; + uniform sampler2D u_texture; + varying vec2 v_texCoord; + void main() { + gl_FragColor = texture2D(u_texture, v_texCoord); + }`; + const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER); + this.gl.shaderSource(fragmentShader, fragmentShaderSource); + this.gl.compileShader(fragmentShader); + + // 创建着色器程序 + const program = this.gl.createProgram(); + this.gl.attachShader(program, vertexShader); + this.gl.attachShader(program, fragmentShader); + this.gl.linkProgram(program); + this.gl.useProgram(program); + + this.rotationMatrixLocation = this.gl.getUniformLocation(program, 'u_rotationMatrix'); + this.textureLocation = this.gl.getUniformLocation(program, 'u_texture'); + + // 创建顶点缓冲区 + const positionBuffer = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, positionBuffer); + this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array([ + -1, -1, + 1, -1, + -1, 1, + 1, 1 + ]), this.gl.STATIC_DRAW); + + // 创建纹理缓冲区 + const texCoordBuffer = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, texCoordBuffer); + this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array([ + 0, 0, + 1, 0, + 0, 1, + 1, 1, + ]), this.gl.STATIC_DRAW); + + // 设置顶点属性指针 + const positionLocation = this.gl.getAttribLocation(program, 'a_position'); + this.gl.enableVertexAttribArray(positionLocation); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, positionBuffer); + this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 0, 0); + + // 设置纹理属性指针 + const texCoordLocation = this.gl.getAttribLocation(program, 'a_texCoord'); + this.gl.enableVertexAttribArray(texCoordLocation); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, texCoordBuffer); + this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 0, 0); + + // 创建纹理对象 + this.texture = this.gl.createTexture(); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture); + + // 设置纹理参数 + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR); + } + + // 每帧更新纹理 + updateTexture() { + let rotateAngles = -90 * Math.PI / 180; + if (this.options.devId === CAMERA_COMMON_MODE_MAP.USER) { + rotateAngles = Math.PI / 2; + } else { + rotateAngles = -90 * Math.PI / 180; + } + // 绑定纹理对象 + this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture); + // 将视频帧绘制到纹理对象 + this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, this.video); + + this.videoEncoderWorker.postMessage({ + type: 'rotateTexture', + rotateAngles + }); + } } \ No newline at end of file diff --git a/sdk/src/worker/VideoEncoder.js b/sdk/src/worker/VideoEncoder.js index 6d3dff237ef20ec5e115707406e2ad8b7341eafc..d7aa0c5086e9fcbc8859cc75aa67a55ac508d317 100644 --- a/sdk/src/worker/VideoEncoder.js +++ b/sdk/src/worker/VideoEncoder.js @@ -1,4 +1,5 @@ import VideoEncode from '../codec/Encode'; +import { mat4 } from 'gl-matrix'; const videoEncoder = new VideoEncode(); @@ -17,6 +18,15 @@ self.addEventListener('message', evt => { videoEncoder.encode(message.data); break; } + case 'rotateTexture': { + let rotationMatrix = mat4.create(); + mat4.rotateZ(rotationMatrix, rotationMatrix, message.rotateAngles); + self.postMessage({ + type: 'setRotateData', + data: rotationMatrix + }); + break; + } default: // do nothing }