From 5649222cb42c06f1da3ff329ae38bfda69b1df9b Mon Sep 17 00:00:00 2001
From: yujiaxinwt <1157613664@qq.com>
Date: Fri, 9 Jun 2023 15:22:12 +0800
Subject: [PATCH 1/2] =?UTF-8?q?=E6=91=84=E5=83=8F=E5=A4=B4=E3=80=81?=
=?UTF-8?q?=E9=BA=A6=E5=85=8B=E9=A3=8E=E9=97=AE=E9=A2=98=E4=BF=AE=E5=A4=8D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
sdk/demo/demo.html | 11 +++-
sdk/index.html | 2 +-
sdk/src/AppController.js | 105 +++++++++++++++++++++------------
sdk/src/CameraPlayer.js | 20 ++++++-
sdk/src/MicrophonePlayer.js | 2 +-
sdk/src/NoDebugger.js | 2 -
sdk/src/Util.js | 12 ++++
sdk/src/config/commonConfig.js | 8 +++
8 files changed, 115 insertions(+), 47 deletions(-)
diff --git a/sdk/demo/demo.html b/sdk/demo/demo.html
index c932f51..29f8acd 100644
--- a/sdk/demo/demo.html
+++ b/sdk/demo/demo.html
@@ -131,6 +131,9 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
selected="selected">720P
+ 信任目标机器
@@ -155,8 +158,8 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
};
var commonLibPath = getConfig.libPath;
var params = {
- ip: 'xx.xx.xx.xx',
- port: 'xx',
+ ip: '123.60.134.174',
+ port: '10181',
session_id: '32uuid',
background_timeout: '60',
available_playtime: '6000',
@@ -167,7 +170,7 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
auto_rotate: true,
libPath: commonLibPath,
decoder_type: 'H264', // H265\H264
- microPhoneOutputType: 'OPUS', // pcm或opus
+ microPhoneOutputType: 'OPUS', // PCM或OPUS
};
if (CloudApp.isSupport()) {
var cloudapp = new CloudApp("container", params);
@@ -210,6 +213,8 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
var typedArray = new Uint8Array(utf8Array);
window.cloudapp.sendDataToCloudApp(typedArray.buffer);
};
+
+ document.getElementById("trustHost").href = 'https://' + [params.ip, params.port].join(':');
}
function exit() {
diff --git a/sdk/index.html b/sdk/index.html
index 5246f9f..189dbbe 100644
--- a/sdk/index.html
+++ b/sdk/index.html
@@ -213,7 +213,7 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
auto_rotate: true,
libPath: commonLibPath, // 配置库文件地址
decoder_type: 'H264', // H265\H264
- microPhoneOutputType: 'OPUS', // pcm或opus
+ microPhoneOutputType: 'OPUS', // PCM或OPUS
media_config: {
bitrate: 3000000,
frame_rate: 30,
diff --git a/sdk/src/AppController.js b/sdk/src/AppController.js
index 87cc59e..18b6c52 100644
--- a/sdk/src/AppController.js
+++ b/sdk/src/AppController.js
@@ -29,6 +29,7 @@ import Util from './Util';
import SocketWorker from './worker/SocketWorker';
import MicrophonePlayer from './MicrophonePlayer';
import CameraPlayer from './CameraPlayer';
+import {MOBILE_CAMERA_MODE_MAP} from './config/commonConfig';
/*global __IS_DEBUG__*/
if (__IS_DEBUG__) {
@@ -81,6 +82,8 @@ const CAMERA_RSP_LEN = MEDIA_MSG_HEADER_COUNT + 2;
const MICROPHONE_RSP_LEN = MEDIA_MSG_HEADER_COUNT + 2;
// 麦克风音频流rsp长度
const MIC_START_RECORD_RSP_LEN = MEDIA_MSG_HEADER_COUNT + 4;
+// pc场景摄像头最大高度
+const PC_CAMERA_MAX_HEIGHT = 720;
class AppController {
constructor(options) {
@@ -98,8 +101,10 @@ class AppController {
'audioStateChange',
'cloudAppData'
]);
- this.mediaMsgHeader = null;
- this.mediaMsgBody = null;
+ this.cameraMsgHeader = null;
+ this.microPhoneMsgHeader = null;
+ this.cameraMsgBody = null;
+ this.microPhoneMsgBody = null;
this.cameraMsgType = PROTOCOL_CONFIG.CAMERA_MESSAGE_TYPE;
this.microPhoneMsgType = PROTOCOL_CONFIG.MICROPHONE_MESSAGE_TYPE;
// WebSocket variable
@@ -456,11 +461,11 @@ class AppController {
}
let buf = new Uint8Array(pkg);
- this.mediaMsgHeader = this.getMsgHeader(buf);
+ this.microPhoneMsgHeader = this.getMsgHeader(buf);
if (buf.length > MEDIA_MSG_HEADER_COUNT) {
- this.mediaMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'MICROPHONE');
+ this.microPhoneMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'MICROPHONE');
}
- const type = this.mediaMsgHeader.optType;
+ const type = this.microPhoneMsgHeader.optType;
switch(type) {
case this.microPhoneMsgType.OPT_MIC_SET_PARAM_REQ:
this.handleSetRecordParam();
@@ -482,11 +487,11 @@ class AppController {
}
let buf = new Uint8Array(pkg);
- this.mediaMsgHeader = this.getMsgHeader(buf);
+ this.cameraMsgHeader = this.getMsgHeader(buf);
if (buf.length > MEDIA_MSG_HEADER_COUNT) {
- this.mediaMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'CAMERA');
+ this.cameraMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'CAMERA');
}
- const type = this.mediaMsgHeader.optType;
+ const type = this.cameraMsgHeader.optType;
switch(type) {
case this.cameraMsgType.OPT_CAMERA_GET_PARAM_REQ:
break;
@@ -511,9 +516,9 @@ class AppController {
sendMediaMsgCmdData(bufLen, msgType, mediaType) {
let rspBuf = new Uint8Array(bufLen);
- this.makeCommonHeaderBuf(rspBuf, msgType, rspBuf.length);
+ this.makeCommonHeaderBuf(rspBuf, msgType, rspBuf.length, mediaType);
// PCM 0 0 ; OPUS 0 1
- if (msgType === this.microPhoneMsgType.OPT_MIC_START_RECORD_RSP && this.options.microPhoneMsgType === 'OPUS') {
+ if (msgType === this.microPhoneMsgType.OPT_MIC_START_RECORD_RSP && this.options.microPhoneOutputType === 'OPUS') {
rspBuf[rspBuf.length - 2] = 1 >> 8;
rspBuf[rspBuf.length - 1] = 1 & 0xFF;
}
@@ -536,11 +541,12 @@ class AppController {
}
handleStartRecordReq() {
+ this.sendMediaMsgCmdData(MIC_START_RECORD_RSP_LEN, this.microPhoneMsgType.OPT_MIC_START_RECORD_RSP, 'MICROPHONE');
this.microphonePlayer = new MicrophonePlayer({
audioOptions: {
sampleRate: 48000,
channelCount: 1,
- maxFrameSize: this.mediaMsgBody?.maxFrameSize || 7680
+ maxFrameSize: this.microPhoneMsgBody?.maxFrameSize || 7680
},
sendOutBufHandler: this.sendMediaData.bind(this),
microPhoneOutputType: this.options.microPhoneOutputType
@@ -548,16 +554,31 @@ class AppController {
}
handleStartPreviewReq() {
+ const originWidth= this.cameraMsgBody.width;
+ const originHeight = this.cameraMsgBody.height;
+ let videoOptions = {
+ width: { ideal: originWidth},
+ height: { ideal: originHeight},
+ frameRate: { ideal: this.cameraMsgBody.fps}
+ };
+ if (originWidth > PC_CAMERA_MAX_HEIGHT && !this.options.isMobile) {
+ let cameraWidth = (originHeight * PC_CAMERA_MAX_HEIGHT) / originWidth;
+ cameraWidth = this.util.changeToMultipleOfEight(cameraWidth);
+ videoOptions.width.ideal = cameraWidth;
+ videoOptions.height.ideal = PC_CAMERA_MAX_HEIGHT;
+ }
+ if (this.options.isMobile) {
+ videoOptions.facingMode = {
+ exact: MOBILE_CAMERA_MODE_MAP[this.cameraMsgHeader.devId]
+ };
+ }
this.cameraPlayer = new CameraPlayer({
- videoOptions: {
- width: { ideal: this.mediaMsgBody.width},
- height: { ideal: this.mediaMsgBody.height},
- frameRate: { ideal: this.mediaMsgBody.fps}
- },
- encodeWidth: this.mediaMsgBody.width,
- encodeHeight: this.mediaMsgBody.height,
+ videoOptions,
+ encodeWidth: this.cameraMsgBody.width,
+ encodeHeight: this.cameraMsgBody.height,
sendOutBufHandler: this.sendMediaData.bind(this),
- x264WasmPath: this.options.libPath
+ x264WasmPath: this.options.libPath,
+ devId: this.cameraMsgHeader.devId
});
}
@@ -720,13 +741,13 @@ class AppController {
/**
*
+ * @param {string} mediaType 媒体类型
* @param {string} msgType 消息类型
- * @param {*} dataType 媒体类型
- * @param {*} data 8位无符号整型数组Unit8Array 帧数据
+ * @param {array} data 8位无符号整型数组Unit8Array 帧数据
*/
- sendMediaData(msgType, dataType, data) {
- let dataBuf = this.makeMediaData(dataType, data);
- let arrayBuf = this.makeDataMsg(msgType, dataBuf);
+ sendMediaData(mediaType, msgType, data) {
+ let dataBuf = this.makeMediaData(msgType, data, mediaType);
+ let arrayBuf = this.makeDataMsg(mediaType, dataBuf);
arrayBuf && this.send(arrayBuf);
}
@@ -822,37 +843,47 @@ class AppController {
* @param {object} headerBuf Uint8Array
* @param {string} rspValue rsp类型，值为protocalConfig配置的key
* @param {number} dataLen 数据总长度
+ * @param {string} mediaType 媒体类型
*/
- makeCommonHeaderBuf(headerBuf, rspValue, dataLen) {
- headerBuf[0] = this.mediaMsgHeader.version >> 8;
- headerBuf[1] = this.mediaMsgHeader.version & 0xFF;
+ makeCommonHeaderBuf(headerBuf, rspValue, dataLen, mediaType) {
+ let currentMsgHeader = null;
+ if (mediaType === 'CAMERA') {
+ currentMsgHeader = this.cameraMsgHeader;
+ }
+ if (mediaType === 'MICROPHONE') {
+ currentMsgHeader = this.microPhoneMsgHeader;
+ }
+
+ headerBuf[0] = currentMsgHeader.version >> 8;
+ headerBuf[1] = currentMsgHeader.version & 0xFF;
headerBuf[2] = rspValue >> 8;
headerBuf[3] = rspValue & 0x00FF;
- headerBuf[4] = this.mediaMsgHeader.devType >> 8;
- headerBuf[5] = this.mediaMsgHeader.devType & 0xFF;
- headerBuf[6] = this.mediaMsgHeader.devId >> 8;
- headerBuf[7] = this.mediaMsgHeader.devId & 0xFF;
+ headerBuf[4] = currentMsgHeader.devType >> 8;
+ headerBuf[5] = currentMsgHeader.devType & 0xFF;
+ headerBuf[6] = currentMsgHeader.devId >> 8;
+ headerBuf[7] = currentMsgHeader.devId & 0xFF;
headerBuf[8] = (dataLen & 0xFF000000) >> 24;
headerBuf[9] = (dataLen & 0x00FF0000) >> 16;
headerBuf[10] = (dataLen & 0x0000FF00) >> 8;
headerBuf[11] = dataLen & 0x000000FF;
- headerBuf[12] = this.mediaMsgHeader.nextProto >> 8;
- headerBuf[13] = this.mediaMsgHeader.nextProto & 0xFF;
- headerBuf[14] = this.mediaMsgHeader.hopLimits >> 8;
- headerBuf[15] = this.mediaMsgHeader.hopLimits & 0xFF;
+ headerBuf[12] = currentMsgHeader.nextProto >> 8;
+ headerBuf[13] = currentMsgHeader.nextProto & 0xFF;
+ headerBuf[14] = currentMsgHeader.hopLimits >> 8;
+ headerBuf[15] = currentMsgHeader.hopLimits & 0xFF;
}
/**
*
* @param {string} dataType 媒体类型：CAMERA、MICROPHONE
* @param {object} data Uint8Array 帧数据
+ * @param {string} mediaType 媒体类型
* @returns
*/
- makeMediaData(dataType, data) {
+ makeMediaData(dataType, data, mediaType) {
let dataBodyLen = data.byteLength;
let dataBuf = new Uint8Array(MEDIA_MSG_HEADER_COUNT + dataBodyLen);
dataBuf.set(new Uint8Array(data), MEDIA_MSG_HEADER_COUNT);
- this.makeCommonHeaderBuf(dataBuf, dataType, dataBuf.length);
+ this.makeCommonHeaderBuf(dataBuf, dataType, dataBuf.length, mediaType);
return dataBuf;
}
diff --git a/sdk/src/CameraPlayer.js b/sdk/src/CameraPlayer.js
index a697859..87db178 100644
--- a/sdk/src/CameraPlayer.js
+++ b/sdk/src/CameraPlayer.js
@@ -1,6 +1,7 @@
import Util from "./Util";
import work from 'webworkify-webpack';
import PROTOCOL_CONFIG from './config/protocolConfig';
+import {CAMERA_COMMON_MODE_MAP} from './config/commonConfig';
const OPT_CAMERA_FRAME = PROTOCOL_CONFIG.CAMERA_MESSAGE_TYPE.OPT_CAMERA_FRAME;
@@ -57,7 +58,20 @@ export default class CameraPlayer {
render() {
const ctx = this.canvas.getContext('2d', {willReadFrequently: true});
- ctx.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
+ ctx.clearRect(0, 0, this.canvas.width,this.canvas.height);
+ ctx.save;
+ if (this.options.devId === CAMERA_COMMON_MODE_MAP.USER) {
+ // 前置摄像头
+ ctx.rotate(Math.PI/2);
+ ctx.drawImage(this.video, 0, -this.canvas.width, this.canvas.height, this.canvas.width);
+
+ } else {
+ // 后置摄像头
+ ctx.rotate(-90 * Math.PI / 180);
+ ctx.drawImage(this.video, -this.canvas.height, 0, this.canvas.height, this.canvas.width);
+ }
+ ctx.restore();
+
const imageData = ctx.getImageData(0, 0, this.canvas.width, this.canvas.height);
this.videoEncoderWorker.postMessage({
@@ -75,8 +89,8 @@ export default class CameraPlayer {
this.canvas.height = this.options.encodeHeight;
this.videoEncoderWorker.postMessage({
type: 'initEncode',
- width: this.options.encodeWidth,
- height: this.options.encodeHeight,
+ width: this.canvas.width,
+ height: this.canvas.height,
x264WasmPath: this.options.x264WasmPath
});
});
diff --git a/sdk/src/MicrophonePlayer.js b/sdk/src/MicrophonePlayer.js
index 19c66ae..4eb9c7e 100644
--- a/sdk/src/MicrophonePlayer.js
+++ b/sdk/src/MicrophonePlayer.js
@@ -64,7 +64,7 @@ export default class MicrophonePlayer {
const inputBuffer = audioProcessEvent.inputBuffer;
const pcmData = inputBuffer.getChannelData(0);
- if (this.options.microPhoneOutputType === 'pcm') {
+ if (this.options.microPhoneOutputType === 'PCM') {
// 转换为16位整数
let initData = new Int16Array(pcmData.length);
for(let i=0; i 10;
}
}
diff --git a/sdk/src/Util.js b/sdk/src/Util.js
index 14c8201..f5fca8d 100644
--- a/sdk/src/Util.js
+++ b/sdk/src/Util.js
@@ -197,6 +197,18 @@ export default class Util {
(array[index + 2] << 8) |
array[index + 3];
}
+
+ /**
+ * @param {number} num 需要转换的数字
+ * @returns 能被8整除的数
+ */
+ changeToMultipleOfEight(num) {
+ if (num % 8 === 0) {
+ return num;
+ } else {
+ return Math.ceil(num / 8) * 8;
+ }
+ }
}
Util.saveAsFile = (() => {
diff --git a/sdk/src/config/commonConfig.js b/sdk/src/config/commonConfig.js
index 7300da7..0f638d5 100644
--- a/sdk/src/config/commonConfig.js
+++ b/sdk/src/config/commonConfig.js
@@ -1,4 +1,12 @@
export const DECODER_TYPE_MAP = {
'H264': 0,
'H265': 1
+}
+export const MOBILE_CAMERA_MODE_MAP = {
+ 1: 'user', // 前置
+ 0: 'environment' // 后置
+}
+export const CAMERA_COMMON_MODE_MAP = {
+ 'USER': 1, // 前置
+ 'ENVIRONMENT': 0 // 后置
}
\ No newline at end of file
--
Gitee
From 4491a37dcd81007a0201f5d3722886d2af80de01 Mon Sep 17 00:00:00 2001
From: yujiaxinwt <1157613664@qq.com>
Date: Fri, 9 Jun 2023 15:30:16 +0800
Subject: [PATCH 2/2] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B5=8B=E8=AF=95?=
=?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=9A=84=E5=9C=B0=E5=9D=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
sdk/demo/demo.html | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sdk/demo/demo.html b/sdk/demo/demo.html
index 29f8acd..e045896 100644
--- a/sdk/demo/demo.html
+++ b/sdk/demo/demo.html
@@ -158,8 +158,8 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
};
var commonLibPath = getConfig.libPath;
var params = {
- ip: '123.60.134.174',
- port: '10181',
+ ip: 'xx.xx.xx.xx',
+ port: 'xx',
session_id: '32uuid',
background_timeout: '60',
available_playtime: '6000',
--
Gitee