diff --git a/sdk/demo/demo.html b/sdk/demo/demo.html
index c932f5105506e5f2884afd07a3ce5fa77cddf745..e045896b112b7ea74d1f1a10310a31590eddcdb1 100644
--- a/sdk/demo/demo.html
+++ b/sdk/demo/demo.html
@@ -131,6 +131,9 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
selected="selected">720P
+ 信任目标机器
@@ -167,7 +170,7 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
auto_rotate: true,
libPath: commonLibPath,
decoder_type: 'H264', // H265\H264
- microPhoneOutputType: 'OPUS', // pcm或opus
+ microPhoneOutputType: 'OPUS', // PCM或OPUS
};
if (CloudApp.isSupport()) {
var cloudapp = new CloudApp("container", params);
@@ -210,6 +213,8 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
var typedArray = new Uint8Array(utf8Array);
window.cloudapp.sendDataToCloudApp(typedArray.buffer);
};
+
+ document.getElementById("trustHost").href = 'https://' + [params.ip, params.port].join(':');
}
function exit() {
diff --git a/sdk/index.html b/sdk/index.html
index 5246f9f364c184b8252cff1f3bc07eb25ced8db0..189dbbef30eba00201a2788d520117edfc0c0798 100644
--- a/sdk/index.html
+++ b/sdk/index.html
@@ -213,7 +213,7 @@ Copyright 2022 Huawei Cloud Computing Technology Co., Ltd.
auto_rotate: true,
libPath: commonLibPath, // 配置库文件地址
decoder_type: 'H264', // H265\H264
- microPhoneOutputType: 'OPUS', // pcm或opus
+ microPhoneOutputType: 'OPUS', // PCM或OPUS
media_config: {
bitrate: 3000000,
frame_rate: 30,
diff --git a/sdk/src/AppController.js b/sdk/src/AppController.js
index 87cc59e775de52baa4e85649d51e0e4f7ba3b36a..18b6c52148dd82ca7d5bbaa9fc2e6f8e9d3b5e3e 100644
--- a/sdk/src/AppController.js
+++ b/sdk/src/AppController.js
@@ -29,6 +29,7 @@ import Util from './Util';
import SocketWorker from './worker/SocketWorker';
import MicrophonePlayer from './MicrophonePlayer';
import CameraPlayer from './CameraPlayer';
+import {MOBILE_CAMERA_MODE_MAP} from './config/commonConfig';
/*global __IS_DEBUG__*/
if (__IS_DEBUG__) {
@@ -81,6 +82,8 @@ const CAMERA_RSP_LEN = MEDIA_MSG_HEADER_COUNT + 2;
const MICROPHONE_RSP_LEN = MEDIA_MSG_HEADER_COUNT + 2;
// 麦克风音频流rsp长度
const MIC_START_RECORD_RSP_LEN = MEDIA_MSG_HEADER_COUNT + 4;
+// pc场景摄像头最大高度
+const PC_CAMERA_MAX_HEIGHT = 720;
class AppController {
constructor(options) {
@@ -98,8 +101,10 @@ class AppController {
'audioStateChange',
'cloudAppData'
]);
- this.mediaMsgHeader = null;
- this.mediaMsgBody = null;
+ this.cameraMsgHeader = null;
+ this.microPhoneMsgHeader = null;
+ this.cameraMsgBody = null;
+ this.microPhoneMsgBody = null;
this.cameraMsgType = PROTOCOL_CONFIG.CAMERA_MESSAGE_TYPE;
this.microPhoneMsgType = PROTOCOL_CONFIG.MICROPHONE_MESSAGE_TYPE;
// WebSocket variable
@@ -456,11 +461,11 @@ class AppController {
}
let buf = new Uint8Array(pkg);
- this.mediaMsgHeader = this.getMsgHeader(buf);
+ this.microPhoneMsgHeader = this.getMsgHeader(buf);
if (buf.length > MEDIA_MSG_HEADER_COUNT) {
- this.mediaMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'MICROPHONE');
+ this.microPhoneMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'MICROPHONE');
}
- const type = this.mediaMsgHeader.optType;
+ const type = this.microPhoneMsgHeader.optType;
switch(type) {
case this.microPhoneMsgType.OPT_MIC_SET_PARAM_REQ:
this.handleSetRecordParam();
@@ -482,11 +487,11 @@ class AppController {
}
let buf = new Uint8Array(pkg);
- this.mediaMsgHeader = this.getMsgHeader(buf);
+ this.cameraMsgHeader = this.getMsgHeader(buf);
if (buf.length > MEDIA_MSG_HEADER_COUNT) {
- this.mediaMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'CAMERA');
+ this.cameraMsgBody = this.getMsgBody(buf.slice(MEDIA_MSG_HEADER_COUNT), 'CAMERA');
}
- const type = this.mediaMsgHeader.optType;
+ const type = this.cameraMsgHeader.optType;
switch(type) {
case this.cameraMsgType.OPT_CAMERA_GET_PARAM_REQ:
break;
@@ -511,9 +516,9 @@ class AppController {
sendMediaMsgCmdData(bufLen, msgType, mediaType) {
let rspBuf = new Uint8Array(bufLen);
- this.makeCommonHeaderBuf(rspBuf, msgType, rspBuf.length);
+ this.makeCommonHeaderBuf(rspBuf, msgType, rspBuf.length, mediaType);
// PCM 0 0 ; OPUS 0 1
- if (msgType === this.microPhoneMsgType.OPT_MIC_START_RECORD_RSP && this.options.microPhoneMsgType === 'OPUS') {
+ if (msgType === this.microPhoneMsgType.OPT_MIC_START_RECORD_RSP && this.options.microPhoneOutputType === 'OPUS') {
rspBuf[rspBuf.length - 2] = 1 >> 8;
rspBuf[rspBuf.length - 1] = 1 & 0xFF;
}
@@ -536,11 +541,12 @@ class AppController {
}
handleStartRecordReq() {
+ this.sendMediaMsgCmdData(MIC_START_RECORD_RSP_LEN, this.microPhoneMsgType.OPT_MIC_START_RECORD_RSP, 'MICROPHONE');
this.microphonePlayer = new MicrophonePlayer({
audioOptions: {
sampleRate: 48000,
channelCount: 1,
- maxFrameSize: this.mediaMsgBody?.maxFrameSize || 7680
+ maxFrameSize: this.microPhoneMsgBody?.maxFrameSize || 7680
},
sendOutBufHandler: this.sendMediaData.bind(this),
microPhoneOutputType: this.options.microPhoneOutputType
@@ -548,16 +554,31 @@ class AppController {
}
handleStartPreviewReq() {
+ const originWidth= this.cameraMsgBody.width;
+ const originHeight = this.cameraMsgBody.height;
+ let videoOptions = {
+ width: { ideal: originWidth},
+ height: { ideal: originHeight},
+ frameRate: { ideal: this.cameraMsgBody.fps}
+ };
+ if (originWidth > PC_CAMERA_MAX_HEIGHT && !this.options.isMobile) {
+ let cameraWidth = (originHeight * PC_CAMERA_MAX_HEIGHT) / originWidth;
+ cameraWidth = this.util.changeToMultipleOfEight(cameraWidth);
+ videoOptions.width.ideal = cameraWidth;
+ videoOptions.height.ideal = PC_CAMERA_MAX_HEIGHT;
+ }
+ if (this.options.isMobile) {
+ videoOptions.facingMode = {
+ exact: MOBILE_CAMERA_MODE_MAP[this.cameraMsgHeader.devId]
+ };
+ }
this.cameraPlayer = new CameraPlayer({
- videoOptions: {
- width: { ideal: this.mediaMsgBody.width},
- height: { ideal: this.mediaMsgBody.height},
- frameRate: { ideal: this.mediaMsgBody.fps}
- },
- encodeWidth: this.mediaMsgBody.width,
- encodeHeight: this.mediaMsgBody.height,
+ videoOptions,
+ encodeWidth: this.cameraMsgBody.width,
+ encodeHeight: this.cameraMsgBody.height,
sendOutBufHandler: this.sendMediaData.bind(this),
- x264WasmPath: this.options.libPath
+ x264WasmPath: this.options.libPath,
+ devId: this.cameraMsgHeader.devId
});
}
@@ -720,13 +741,13 @@ class AppController {
/**
*
+ * @param {string} mediaType 媒体类型
* @param {string} msgType 消息类型
- * @param {*} dataType 媒体类型
- * @param {*} data 8位无符号整型数组Unit8Array 帧数据
+ * @param {array} data 8位无符号整型数组Unit8Array 帧数据
*/
- sendMediaData(msgType, dataType, data) {
- let dataBuf = this.makeMediaData(dataType, data);
- let arrayBuf = this.makeDataMsg(msgType, dataBuf);
+ sendMediaData(mediaType, msgType, data) {
+ let dataBuf = this.makeMediaData(msgType, data, mediaType);
+ let arrayBuf = this.makeDataMsg(mediaType, dataBuf);
arrayBuf && this.send(arrayBuf);
}
@@ -822,37 +843,47 @@ class AppController {
* @param {object} headerBuf Uint8Array
* @param {string} rspValue rsp类型，值为protocalConfig配置的key
* @param {number} dataLen 数据总长度
+ * @param {string} mediaType 媒体类型
*/
- makeCommonHeaderBuf(headerBuf, rspValue, dataLen) {
- headerBuf[0] = this.mediaMsgHeader.version >> 8;
- headerBuf[1] = this.mediaMsgHeader.version & 0xFF;
+ makeCommonHeaderBuf(headerBuf, rspValue, dataLen, mediaType) {
+ let currentMsgHeader = null;
+ if (mediaType === 'CAMERA') {
+ currentMsgHeader = this.cameraMsgHeader;
+ }
+ if (mediaType === 'MICROPHONE') {
+ currentMsgHeader = this.microPhoneMsgHeader;
+ }
+
+ headerBuf[0] = currentMsgHeader.version >> 8;
+ headerBuf[1] = currentMsgHeader.version & 0xFF;
headerBuf[2] = rspValue >> 8;
headerBuf[3] = rspValue & 0x00FF;
- headerBuf[4] = this.mediaMsgHeader.devType >> 8;
- headerBuf[5] = this.mediaMsgHeader.devType & 0xFF;
- headerBuf[6] = this.mediaMsgHeader.devId >> 8;
- headerBuf[7] = this.mediaMsgHeader.devId & 0xFF;
+ headerBuf[4] = currentMsgHeader.devType >> 8;
+ headerBuf[5] = currentMsgHeader.devType & 0xFF;
+ headerBuf[6] = currentMsgHeader.devId >> 8;
+ headerBuf[7] = currentMsgHeader.devId & 0xFF;
headerBuf[8] = (dataLen & 0xFF000000) >> 24;
headerBuf[9] = (dataLen & 0x00FF0000) >> 16;
headerBuf[10] = (dataLen & 0x0000FF00) >> 8;
headerBuf[11] = dataLen & 0x000000FF;
- headerBuf[12] = this.mediaMsgHeader.nextProto >> 8;
- headerBuf[13] = this.mediaMsgHeader.nextProto & 0xFF;
- headerBuf[14] = this.mediaMsgHeader.hopLimits >> 8;
- headerBuf[15] = this.mediaMsgHeader.hopLimits & 0xFF;
+ headerBuf[12] = currentMsgHeader.nextProto >> 8;
+ headerBuf[13] = currentMsgHeader.nextProto & 0xFF;
+ headerBuf[14] = currentMsgHeader.hopLimits >> 8;
+ headerBuf[15] = currentMsgHeader.hopLimits & 0xFF;
}
/**
*
* @param {string} dataType 媒体类型：CAMERA、MICROPHONE
* @param {object} data Uint8Array 帧数据
+ * @param {string} mediaType 媒体类型
* @returns
*/
- makeMediaData(dataType, data) {
+ makeMediaData(dataType, data, mediaType) {
let dataBodyLen = data.byteLength;
let dataBuf = new Uint8Array(MEDIA_MSG_HEADER_COUNT + dataBodyLen);
dataBuf.set(new Uint8Array(data), MEDIA_MSG_HEADER_COUNT);
- this.makeCommonHeaderBuf(dataBuf, dataType, dataBuf.length);
+ this.makeCommonHeaderBuf(dataBuf, dataType, dataBuf.length, mediaType);
return dataBuf;
}
diff --git a/sdk/src/CameraPlayer.js b/sdk/src/CameraPlayer.js
index a6978598e3d6652d052e37431d4e1505672b4ace..87db178b31e307734adfda758a65c0433b1fef3a 100644
--- a/sdk/src/CameraPlayer.js
+++ b/sdk/src/CameraPlayer.js
@@ -1,6 +1,7 @@
import Util from "./Util";
import work from 'webworkify-webpack';
import PROTOCOL_CONFIG from './config/protocolConfig';
+import {CAMERA_COMMON_MODE_MAP} from './config/commonConfig';
const OPT_CAMERA_FRAME = PROTOCOL_CONFIG.CAMERA_MESSAGE_TYPE.OPT_CAMERA_FRAME;
@@ -57,7 +58,20 @@ export default class CameraPlayer {
render() {
const ctx = this.canvas.getContext('2d', {willReadFrequently: true});
- ctx.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
+ ctx.clearRect(0, 0, this.canvas.width,this.canvas.height);
+ ctx.save;
+ if (this.options.devId === CAMERA_COMMON_MODE_MAP.USER) {
+ // 前置摄像头
+ ctx.rotate(Math.PI/2);
+ ctx.drawImage(this.video, 0, -this.canvas.width, this.canvas.height, this.canvas.width);
+
+ } else {
+ // 后置摄像头
+ ctx.rotate(-90 * Math.PI / 180);
+ ctx.drawImage(this.video, -this.canvas.height, 0, this.canvas.height, this.canvas.width);
+ }
+ ctx.restore();
+
const imageData = ctx.getImageData(0, 0, this.canvas.width, this.canvas.height);
this.videoEncoderWorker.postMessage({
@@ -75,8 +89,8 @@ export default class CameraPlayer {
this.canvas.height = this.options.encodeHeight;
this.videoEncoderWorker.postMessage({
type: 'initEncode',
- width: this.options.encodeWidth,
- height: this.options.encodeHeight,
+ width: this.canvas.width,
+ height: this.canvas.height,
x264WasmPath: this.options.x264WasmPath
});
});
diff --git a/sdk/src/MicrophonePlayer.js b/sdk/src/MicrophonePlayer.js
index 19c66ae484f872313562a9ff3f0d2bc81f173c86..4eb9c7eb0e829047d402a9aaa7af564982db5dc5 100644
--- a/sdk/src/MicrophonePlayer.js
+++ b/sdk/src/MicrophonePlayer.js
@@ -64,7 +64,7 @@ export default class MicrophonePlayer {
const inputBuffer = audioProcessEvent.inputBuffer;
const pcmData = inputBuffer.getChannelData(0);
- if (this.options.microPhoneOutputType === 'pcm') {
+ if (this.options.microPhoneOutputType === 'PCM') {
// 转换为16位整数
let initData = new Int16Array(pcmData.length);
for(let i=0; i 10;
}
}
diff --git a/sdk/src/Util.js b/sdk/src/Util.js
index 14c820166fef8aea256e2cfcce12099a8ebfc094..f5fca8db6e1937f1d179bdfdea92f52a92b9ad6a 100644
--- a/sdk/src/Util.js
+++ b/sdk/src/Util.js
@@ -197,6 +197,18 @@ export default class Util {
(array[index + 2] << 8) |
array[index + 3];
}
+
+ /**
+ * @param {number} num 需要转换的数字
+ * @returns 能被8整除的数
+ */
+ changeToMultipleOfEight(num) {
+ if (num % 8 === 0) {
+ return num;
+ } else {
+ return Math.ceil(num / 8) * 8;
+ }
+ }
}
Util.saveAsFile = (() => {
diff --git a/sdk/src/config/commonConfig.js b/sdk/src/config/commonConfig.js
index 7300da7a106efac872d9d1b8611be07893a6934f..0f638d5a628d3b0080cb31ac37a16071bec26fe8 100644
--- a/sdk/src/config/commonConfig.js
+++ b/sdk/src/config/commonConfig.js
@@ -1,4 +1,12 @@
export const DECODER_TYPE_MAP = {
'H264': 0,
'H265': 1
+}
+export const MOBILE_CAMERA_MODE_MAP = {
+ 1: 'user', // 前置
+ 0: 'environment' // 后置
+}
+export const CAMERA_COMMON_MODE_MAP = {
+ 'USER': 1, // 前置
+ 'ENVIRONMENT': 0 // 后置
}
\ No newline at end of file