diff --git a/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts new file mode 100644 index 0000000000000000000000000000000000000000..807afde2f2a78de529a40540f458e9ca38ef8073 --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts @@ -0,0 +1,31 @@ +// declgen/declgen_worker.ts +import { parentPort } from 'worker_threads'; +import * as fs from 'fs'; +import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; +import { processInteropUI } from '../../../process_interop_ui'; +import type { RunnerParms } from './type'; + +if (!parentPort) { + throw new Error('This file should be run as a Worker'); +} + +parentPort.on('message', async (config: RunnerParms) => { + try { + if (!fs.existsSync(config.outDir)) { + fs.mkdirSync(config.outDir, { recursive: true }); + } + + generateInteropDecls(config); + + if (config.outDir) { + processInteropUI(config.outDir); + } + + parentPort?.postMessage({ status: 'done' }); + } catch (error: any) { + parentPort?.postMessage({ + status: 'error', + error: error?.message || String(error) + }); + } +}); diff --git a/compiler/src/fast_build/ark_compiler/interop/main.ts b/compiler/src/fast_build/ark_compiler/interop/main.ts new file mode 100644 index 0000000000000000000000000000000000000000..278068a6fcbf466ec902baa2825171842f169e3c --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/main.ts @@ -0,0 +1,223 @@ +// let param = { +// tasks: [ +// { +// packageName: "application", +// buildModule: "declgen" +// } +// ], +// dependentModuleMap: { +// 'application': { +// language: '1.1', +// packageName: 'application', +// moduleName: 'entry', +// modulePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application', +// declgenV1OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\declgenV1', +// declgenV2OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\declgenV2', +// declgenBridgeCodePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', +// declFilesPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', +// fileList: [ +// "D:/temp/helloworld_0331/helloworld_0331/application/src/main/ets/pages/Index.ets" +// ], +// cachePath:"D:\\outDemo", +// byteCodeHarInfo: { +// bchar: { +// abcPath: "D:\\modules.abc", +// } +// }, +// pkgPath:'D:\\temp\\helloworld_0331\\helloworld_0331\\application', +// packageVersion:'1.0.0' +// }, +// 'library': { +// language: '1.2', +// packageName: 'library', +// moduleName: 'library', +// pkgPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library', +// modulePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library', +// pkgPath:'D:\\temp\\helloworld_0331\\helloworld_0331\\library', +// declgenV1OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\declgenV1', +// declgenV2OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\declgenV2', +// declgenBridgeCodePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', +// declFilesPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', +// fileList: [ +// "D:/temp/helloworld_0331/helloworld_0331/application/src/main/ets/pages/Index.ets" +// ] +// } +// }, +// projectConfig:{ +// bundleName: "application", +// mainModuleName: "mainModuleName", +// projectRootPath: "D:/temp/helloworld_0331/helloworld_0331", +// } +// } +// let param = { +// tasks: [ +// {packageName: 'library', buildTask: 'declgen'}, +// {packageName: 'mixhar', buildTask: 'declgen'} +// ], +// dependentModuleMap: new Map([ +// ['library', { +// language: '1.1', +// packageName: 'library', +// moduleName: 'library', +// modulePath: 'D:\\\\projectTest\\\\MyApplication38\\\\library', +// declgenV2OutPath: 'D:\\\\projectTest\\\\MyApplication38\\\\library\\\\build\\\\default\\\\intermediates\\\\declgen\\\\default\\\\declgenV2', +// dynamicFiles:[ +// 'D:\\projectTest\\MyApplication38\\library\\Index.ets', +// 'D:\\projectTest\\MyApplication38\\library\\src\\main\\ets\\components\\cccc.ets', +// 'D:\\projectTest\\MyApplication38\\library\\src\\main\\ets\\components\\MainPage.ets' +// ], +// cachePath:'D:\\projectTest\\MyApplication38\\library\\build\\default\\intermediates\\declgen\\default\\cache', +// pkgPath: 'D:\\projectTest\\MyApplication38\\library', +// packageVersion: '1.0.0' +// }], +// ['mixhar', { +// language: 'hybrid', +// packageName: 'mixhar', +// moduleName: 'mixHar', +// modulePath: 'D:\\projectTest\\MyApplication38\\mixHar', +// declFilesPath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', +// declgenV1OutPath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\declgenV1', +// declgenV2OutPath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\declgenV2', +// declgenBridgeCodePath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', +// dynamicFiles: [ +// 'D:\\projectTest\\MyApplication38\\mixHar\\Index.ets' +// ], +// cachePath: 'D:\\projectTest\\MyApplication38\\build\\declgen\\mixhar\\cache', +// pkgPath: 'D:\\projectTest\\MyApplication38\\mixHar', +// packageVersion: '1.0.0' +// }] +// ]) +// , +// projectConfig: { +// bundleName: 'com.example.myapplication', +// projectRootPath: 'D:\\projectTest\\MyApplication38' +// } +// } + +let declgen = require("D:\\SDK\\OH\\20\\ets\\ets1.1\\build-tools\\ets-loader\\lib\\fast_build\\ark_compiler\\interop\\run_declgen_standalone.js"); + +let param = { + tasks: [ + { + packageName: "entry", + buildTask: "declgen", + mainModuleName: "entry" + }, + { + packageName: "@alipay/library1", + buildTask: "declgen", + mainModuleName: "entry" + }, + { + packageName: "@alipay/library2", + buildTask: "declgen", + mainModuleName: "entry" + } + ], + dependentModuleMap: new Map([ + ["entry", { + language: "hybrid", + packageName: "entry", + moduleName: "entry", + modulePath: "D:\\code\\openharmony\\mytest\\my-test\\entry", + declFilesPath: "D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json", + declgenV1OutPath: "D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenV1", + declgenV2OutPath: "D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenV2", + declgenBridgeCodePath: "D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode", + dynamicFiles: [ + "D:\\code\\openharmony\\mytest\\my-test\\entry\\src\\main\\ets\\entryability\\EntryAbility.ets", + "D:\\code\\openharmony\\mytest\\my-test\\entry\\src\\main\\ets\\pages\\Index.ets" + ], + staticFiles: [], + cachePath: "D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\cache", + byteCodeHarInfo: { + "entry": { + abcPath: "D:\\code\\openharmony\\mytest\\my-test\\entry\\ets\\modules.abc" + } + }, + pkgPath: "D:\\code\\openharmony\\mytest\\my-test\\entry", + packageVersion: "1.0.0" + }], + ["@alipay/library1", { + language: "hybrid", + packageName: "@alipay/library1", + moduleName: "library1", + modulePath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1", + declFilesPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json", + declgenV1OutPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenV1", + declgenV2OutPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenV2", + declgenBridgeCodePath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode", + dynamicFiles: [ + "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\Index.ets", + "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\src\\main\\ets\\components\\MainPage.ets" + ], + staticFiles: [], + cachePath: "D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\library1\\cache", + byteCodeHarInfo: { + "@alipay/library1": { + "abcPath": "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\ets\\modules.abc" + } + }, + pkgPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library1", + packageVersion: "1.0.0" + }], + ["@alipay/library2", { + language: "hybrid", + packageName: "@alipay/library2", + moduleName: "library2", + modulePath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2", + declFilesPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json", + declgenV1OutPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenV1", + declgenV2OutPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenV2", + declgenBridgeCodePath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode", + dynamicFiles: [ + "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\Index.ets", + "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\src\\main\\ets\\components\\MainPage.ets" + ], + staticFiles: [], + cachePath: "D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\library2\\cache", + byteCodeHarInfo: { + "@alipay/library2": { + "abcPath": "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\ets\\modules.abc" + } + }, + pkgPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\library2", + packageVersion: "1.0.0" + }], + ["@alipay/librarystatic1", { + language: "hybrid", + packageName: "@alipay/librarystatic1", + moduleName: "libraryStatic1", + modulePath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1", + declFilesPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json", + declgenV1OutPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\declgenV1", + declgenV2OutPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\declgenV2", + declgenBridgeCodePath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode", + dynamicFiles: [], + staticFiles: [ + "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\Index.ets", + "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\src\\main\\ets\\components\\MainPage.ets" + ], + cachePath: "D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\librarystatic1\\cache", + byteCodeHarInfo: { + "@alipay/librarystatic1": { + "abcPath": "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\ets\\modules.abc" + } + }, + pkgPath: "D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1", + packageVersion: "1.0.0" + }] + ]), + projectConfig: { + "bundleName": "com.example.alipaytest", + "projectRootPath": "D:\\code\\openharmony\\mytest\\my-test" + } +} + +async function runDeclgen() { + await declgen.run(param); + + console.log('wait until finish'); +} + +runDeclgen().catch(err => console.error("Error:", err)); diff --git a/compiler/src/fast_build/ark_compiler/interop/record_time_mem.ts b/compiler/src/fast_build/ark_compiler/interop/record_time_mem.ts new file mode 100644 index 0000000000000000000000000000000000000000..fb8df05c7a0e8e1bc735bf572e9063be2708f30f --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/record_time_mem.ts @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import path from 'path'; +import { RECORD_TYPE } from './type' + +export class TimeMemData { + public startTime: number = 0; + public endtime: number = 0; + public time: number = 0; + public startMem: number = 0; + public endMem:number = 0; + public mem: number = 0; +} + +export class SingleData { + public time: number = 0; + public mem: number = 0; +} + +export class CompileSingleData { + private timeMemMap: Map; + private totalTime: number = 0; + private startTime: number = 0; + private startMem: number = 0; + private file: string = ''; + private recordType: RECORD_TYPE; + + constructor(file: string, recordType: RECORD_TYPE) { + this.file = file; + this.timeMemMap = new Map(); + this.recordType = recordType; + } + + public record(startKey: string, lastEndKey: string = '') { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + let currentTime = new Date().getTime(); + let currentMem = process.memoryUsage.rss(); + let tmp: SingleData | undefined = this.timeMemMap.get(lastEndKey); + if (tmp) { + tmp.time = currentTime - this.startTime; + tmp.mem = (currentMem > this.startMem) ? (currentMem- this.startMem) : 0; ; + this.timeMemMap.set(lastEndKey, tmp); + } + + if (startKey == '') { + return; + } + + let tmp1: SingleData | undefined = this.timeMemMap.get(startKey); + if (tmp1 == undefined) { + this.startTime = currentTime; + this.startMem = currentMem; + let data: SingleData = new SingleData(); + data.time = 0; + data.mem = 0; + this.timeMemMap.set(startKey, data); + } + } + + writeSumSingle(cachePath: string, deputyName: string = '') { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + const csvData: string[] = [ + "timeKey, time(ms), mem(M)" + ]; + this.timeMemMap.forEach((v: SingleData, k: string) => { + let element = `${k}` +', ' + `${v.time}` + 'ms' + ', ' + `${Math.round(v.mem / 1024 / 1024)}` + 'M' ; + csvData.push(element); + }); + let name = path.basename(this.file) + let currentExt = path.extname(name) + let fileWithoutExt = name.substring(0, name.lastIndexOf(currentExt)); + let fileName = `${fileWithoutExt}`+ deputyName +'_time_mem.csv'; + let filePath = path.join(cachePath, fileName); + csvData.forEach(row => { + fs.appendFileSync(filePath, `${row}\n`); + }); + } +} + + +export class RecordTimeMem { + private static instance: RecordTimeMem | undefined; + private compileType: string = ''; + private memLow: number = 0; + private memHigh: number = 0; + private timeMemMap: Map; + private recordType: RECORD_TYPE = RECORD_TYPE.DEFAULT_TYPE;; + + public static getInstance(): RecordTimeMem { + if (!RecordTimeMem.instance) { + RecordTimeMem.instance = new RecordTimeMem(); + } + return RecordTimeMem.instance; + } + + private constructor() { + this.memLow = process.memoryUsage.rss(); + this.timeMemMap = new Map(); + } + + public setCompileType(type: string) { + this.compileType = type; + } + + public initRecordType(recordType: RECORD_TYPE) { + this.recordType = recordType; + } + + public start(timeKey: string) { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + let currentTime = new Date().getTime(); + let currentMem = process.memoryUsage.rss(); + let tmp: TimeMemData | undefined = this.timeMemMap.get(timeKey); + if (tmp) { + tmp.startTime = currentTime; + tmp.startMem = currentMem; + this.timeMemMap.set(timeKey, tmp); + } else { + let data: TimeMemData = new TimeMemData(); + data.startTime = currentTime; + data.endMem = currentMem; + this.timeMemMap.set(timeKey, data); + } + this.recordMem(); + } + + public end(timeKey: string) { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + let currentTime = new Date().getTime(); + let currentMem = process.memoryUsage.rss(); + let tmp: TimeMemData | undefined = this.timeMemMap.get(timeKey); + if (tmp) { + tmp.endtime = currentTime; + tmp.endMem = currentMem; + tmp.time = tmp.endtime - tmp.startTime; + tmp.mem = (tmp.endMem > tmp.startMem) ? (tmp.endMem - tmp.startMem) : 0; + this.timeMemMap.set(timeKey, tmp); + } + this.recordMem(); + } + + + private recordMem() { + let rss: number = process.memoryUsage.rss(); + if (rss < this.memLow) { + this.memLow = rss; + } + if (rss > this.memHigh) { + this.memHigh = rss; + } + } + + writeDataCsv(filePath: string, data: any[]) { + const content = data.map(row => { + return Object.values(row).join(','); + }).join('\n'); + fs.writeFile(filePath, content, (err) => { + if (err) { + console.error('Error writing file:', err); + } + }); + } + + writeSum(cachePath: string) { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + const csvData: any[] = []; + csvData.push({ timeKey: `memory low`, time: 'undefined', mem: `${Math.round(this.memLow / 1024 / 1024)}` + 'M' }); + csvData.push({ timeKey: `memory high`, time: 'undefined', mem: `${Math.round(this.memHigh / 1024 / 1024)}` + 'M' }); + csvData.push({ timeKey: `timeKey`, time: `time` + '(ms)', mem: `mem` + '(M)' }); + this.timeMemMap.forEach((v: TimeMemData, k: string) => { + let element = { timeKey: `${k}`, time: `${v.time}` + 'ms', mem: `${Math.round(v.mem / 1024 / 1024)}` + 'M' }; + csvData.push(element); + }) + let fileName = `${this.compileType}`+'_time_mem.csv'; + let filePath = path.join(cachePath, fileName); + this.writeDataCsv(filePath, csvData); + } +} \ No newline at end of file diff --git a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts index b1cb71d67c0e57c03a45807bacd80068360b9664..ed61c5108fbf17b74328981b2af5deb70695a538 100644 --- a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts +++ b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts @@ -37,27 +37,62 @@ import { EXTNAME_D_ETS, EXTNAME_JS } from '../common/ark_define'; import { getRealModulePath } from '../../system_api/api_check_utils'; import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; import { calculateFileHash } from '../utils'; +import { runWithWorkerPool } from './thread_pool'; + +export async function run(param: Params): Promise { +// console.log('something special'); + FileManager.init(param.dependentModuleMap); + DeclfileProductor.init(param); +// console.log('stringify', JSON.stringify(param)); +// console.log('stringifyMap', JSON.stringify(Object.fromEntries(param.dependentModuleMap.entries()))); + + const declgenTasks: ArkTSEvolutionModule[] = []; + + for (const task of param.tasks) { + const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); + if (task.buildTask === BuildType.DECLGEN) { + declgenTasks.push(moduleInfo); + } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { + DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); + } + } + + console.log('current tasks', declgenTasks.length); + await runWithWorkerPool( + declgenTasks, + 4, + (moduleInfo) => DeclfileProductor.getInstance().getWorkerPayload(moduleInfo), + (moduleInfo, msg) => { + if (msg.status === 'error') { + console.error(`Declgen failed for ${moduleInfo.packageName}:`, msg.error); + } + } + ); -export function run(param: Params): boolean { - FileManager.init(param.dependentModuleMap); - DeclfileProductor.init(param); - param.tasks.forEach(task => { - const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); - if (moduleInfo?.dynamicFiles.length <= 0) { - return; - } - if (task.buildTask === BuildType.DECLGEN) { - DeclfileProductor.getInstance().runDeclgen(moduleInfo); - } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { - DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); - } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { - //todo - } - }); - FileManager.cleanFileManagerObject(); - return true; + FileManager.cleanFileManagerObject(); + return true; } +// export function run(param: Params): boolean { +// FileManager.init(param.dependentModuleMap); +// DeclfileProductor.init(param); +// param.tasks.forEach(task => { +// const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); +// if (moduleInfo?.dynamicFiles.length <= 0) { +// return; +// } +// if (task.buildTask === BuildType.DECLGEN) { +// DeclfileProductor.getInstance().runDeclgen(moduleInfo); +// } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { +// DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); +// } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { +// //todo +// } +// }); +// FileManager.cleanFileManagerObject(); +// return true; +// } + class DeclfileProductor { private static declFileProductor: DeclfileProductor; @@ -255,6 +290,36 @@ class DeclfileProductor { ]; DeclfileProductor.sdkConfigs = [...DeclfileProductor.defaultSdkConfigs]; } + + getWorkerPayload(moduleInfo: ArkTSEvolutionModule): any { + const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; + let existingCache = {}; + const filesToProcess: string[] = []; + const hashMap = {}; + + if (fs.existsSync(cachePath)) { + existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); + } + + moduleInfo.dynamicFiles.forEach(file => { + const unixPath = file.replace(/\\/g, '/'); + const hash = calculateFileHash(file); + if (!existingCache[unixPath] || existingCache[unixPath] !== hash) { + filesToProcess.push(unixPath); + hashMap[unixPath] = hash; + } + }); + + return { + inputDirs: [], + inputFiles: filesToProcess, + outDir: moduleInfo.declgenV2OutPath, + rootDir: moduleInfo.modulePath, + customResolveModuleNames: null, + customCompilerOptions: DeclfileProductor.compilerOptions, + includePaths: [moduleInfo.modulePath] + }; + } } function resolveModuleNames(moduleNames: string[], containingFile: string): ts.ResolvedModuleFull[] { diff --git a/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts new file mode 100644 index 0000000000000000000000000000000000000000..0023b17f04a3096066accfe65d3674ce6650da4d --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts @@ -0,0 +1,223 @@ +import { Worker } from 'worker_threads'; +import * as path from 'path'; + +interface WorkerTask { + task: T; + resolve: () => void; + reject: (reason?: any) => void; +} + +class WorkerPool { + private idleWorkers: Worker[] = []; + private activeWorkers = 0; + private taskQueue: WorkerTask[] = []; + private workerPath: string; + private getPayload: (task: T) => any; + private onMessage?: (task: T, msg: any) => void; + + constructor( + maxWorkers: number, + workerPath: string, + getPayload: (task: T) => any, + onMessage?: (task: T, msg: any) => void + ) { + this.workerPath = workerPath; + this.getPayload = getPayload; + this.onMessage = onMessage; + + // 预热线程池 + for (let i = 0; i < maxWorkers; i++) { + this.addWorker(); + } + } + + private addWorker() { + const worker = new Worker(this.workerPath); + worker.unref(); // 允许进程退出 + this.idleWorkers.push(worker); + } + + enqueue(task: T): Promise { + return new Promise((resolve, reject) => { + this.taskQueue.push({ task, resolve, reject }); + this.processQueue(); + }); + } + + private processQueue() { + while (this.idleWorkers.length > 0 && this.taskQueue.length > 0) { + const worker = this.idleWorkers.shift()!; + const taskItem = this.taskQueue.shift()!; + this.executeTask(worker, taskItem); + } + } + + private executeTask(worker: Worker, taskItem: WorkerTask) { + this.activeWorkers++; + + const { task, resolve, reject } = taskItem; + const payload = this.getPayload(task); + + // 清理旧监听器 + worker.removeAllListeners(); + + // 设置新监听器 + worker.on('message', (msg) => { + if (this.onMessage) { + this.onMessage(task, msg); + } + + if (msg.status === 'done') { + cleanup(); + resolve(); + this.returnWorker(worker); + } else if (msg.status === 'error') { + cleanup(); + reject(new Error(msg.error || 'Worker error')); + this.returnWorker(worker); + } + }); + + worker.on('error', (err) => { + cleanup(); + console.error(`Worker error: ${err.message}`); + reject(err); + this.replaceWorker(worker); + }); + + worker.on('exit', (code) => { + if (code !== 0) { + cleanup(); + reject(new Error(`Worker stopped with exit code ${code}`)); + this.replaceWorker(worker); + } + }); + + const cleanup = () => { + worker.removeAllListeners(); + this.activeWorkers--; + }; + + try { + worker.postMessage(payload); + } catch (err) { + cleanup(); + reject(err); + this.replaceWorker(worker); + } + } + + private returnWorker(worker: Worker) { + this.idleWorkers.push(worker); + this.processQueue(); + } + + private replaceWorker(worker: Worker) { + worker.terminate(); + this.addWorker(); + this.processQueue(); + } + + async drain() { + while (this.activeWorkers > 0 || this.taskQueue.length > 0) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } +} + +// export async function runWithWorkerPool( +// tasks: T[], +// maxConcurrency: number, +// getWorkerPayload: (task: T) => any, +// onWorkerMessage?: (task: T, msg: any) => void +// ): Promise { +// const workerPath = path.resolve(__dirname, './declgen_worker.js'); +// const pool = new WorkerPool( +// maxConcurrency, +// workerPath, +// getWorkerPayload, +// onWorkerMessage +// ); + +// // 添加任务到队列 +// const taskPromises = tasks.map(task => pool.enqueue(task)); + +// try { +// await Promise.all(taskPromises); +// } catch (error) { +// console.error('Error in worker pool:', error); +// throw error; +// } finally { +// // 等待所有任务完成 +// await pool.drain(); +// } +// } + +/** + * @param tasks 任务列表 + * @param maxConcurrency 最大并发线程数 + * @param getWorkerPayload 将任务对象转换为传给 Worker 的数据 + * @param onWorkerMessage 可选：处理 Worker 返回的消息 + */ +export async function runWithWorkerPool( + tasks: T[], + maxConcurrency: number, + getWorkerPayload: (task: T) => any, + onWorkerMessage?: (task: T, msg: any) => void +): Promise { + const queue = [...tasks]; + const activeWorkers: Promise[] = []; + + const runWorker = async (task: T): Promise => { + return new Promise((resolve, reject) => { + const workerPath = path.resolve(__dirname, './declgen_worker.js'); + const worker = new Worker(workerPath); + + const payload = getWorkerPayload(task); + worker.postMessage(payload); + + worker.on('message', (msg: any) => { + if (onWorkerMessage) { + onWorkerMessage(task, msg); + } + if (msg.status === 'done') { + worker.terminate(); // 终止线程 + resolve(); + console.log('done and terminate'); + } else if (msg.status === 'error') { + reject(new Error(msg.error || 'Worker encountered an error')); + console.log('Worker encountered an error', msg.error); + } + }); + + worker.on('error', (err) => { + reject(err); + console.log('worker error', err); + }); + + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker stopped with exit code ${code}`)); + } + console.log('worker exit', code); + }); + }); + }; + + for (let i = 0; i < maxConcurrency; i++) { + const loop = async () => { + while (queue.length > 0) { + const task = queue.shift(); + if (task) { + console.log('processing task'); + await runWorker(task).catch((err) => { + console.error(`[thread_pool] Worker task failed:`, err.message); + }); + } + } + }; + activeWorkers.push(loop()); + } + + await Promise.all(activeWorkers); +}