From 6a75b738c391840e466b2571704674362bbc6db7 Mon Sep 17 00:00:00 2001 From: alichinese Date: Sun, 28 Apr 2024 15:02:21 +0800 Subject: [PATCH] compile: Implement one-click building * Build recipes directly through oebuild + yaml Signed-off-by: alichinese --- src/oebuild/app/main.py | 167 +++++++++++++++++- src/oebuild/app/plugins/generate/generate.py | 9 +- src/oebuild/app/plugins/init/init.py | 3 +- .../app/plugins/toolchain/toolchain.py | 7 +- src/oebuild/app/plugins/update/update.py | 2 +- src/oebuild/bashrc.py | 13 +- src/oebuild/local_conf.py | 8 +- src/oebuild/parse_param.py | 45 ++++- src/oebuild/parse_template.py | 2 +- src/oebuild/struct.py | 24 ++- src/oebuild/util.py | 3 + 11 files changed, 245 insertions(+), 38 deletions(-) diff --git a/src/oebuild/app/main.py b/src/oebuild/app/main.py index ac10b47..80aae9b 100644 --- a/src/oebuild/app/main.py +++ b/src/oebuild/app/main.py @@ -18,11 +18,14 @@ import getpass import colorama import oebuild.util as oebuild_util +import oebuild.const as oebuild_const +from oebuild.m_log import logger +from oebuild.struct import CompileParam +from oebuild.parse_param import ParseOebuildEnvParam, ParseCompileParam from oebuild.auto_completion import AutoCompletion from oebuild.version import __version__ from oebuild.spec import get_spec, _ExtCommand from oebuild.command import OebuildCommand -from oebuild.m_log import logger from oebuild.oebuild_parser import OebuildArgumentParser, OebuildHelpAction APP = "app" @@ -41,13 +44,13 @@ class OebuildApp: self.cmd = None try: plugins_dir = pathlib.Path(self.base_oebuild_dir, 'app/conf', 'plugins.yaml') - self.oebuild_plugins_path = os.path.expanduser('~') + '/.local/oebuild_plugins/' - self.append_plugins_dir = pathlib.Path(self.oebuild_plugins_path, 'append_plugins.yaml') + oebuild_plugins_path = os.path.expanduser('~') + '/.local/oebuild_plugins/' + append_plugins_dir = pathlib.Path(oebuild_plugins_path, 'append_plugins.yaml') self.command_ext = self.get_command_ext(oebuild_util.read_yaml(plugins_dir)['plugins']) - if os.path.exists(self.append_plugins_dir) \ - and oebuild_util.read_yaml(self.append_plugins_dir): + if os.path.exists(append_plugins_dir) \ + and oebuild_util.read_yaml(append_plugins_dir): self.command_ext = self.get_command_ext( - oebuild_util.read_yaml(self.append_plugins_dir)['plugins'], + oebuild_util.read_yaml(append_plugins_dir)['plugins'], self.command_ext) self.command_spec = {} except Exception as e_p: @@ -188,6 +191,150 @@ def extension_commands(pre_dir, commandlist: OrderedDict): return specs +class QuickBuild(): + ''' + The build command will quickly generate the compile.yaml + ''' + + def __init__(self, build_yaml_path): + self.app = OebuildApp() + self.build_yaml_path = pathlib.Path(build_yaml_path) + self.oebuild_env = None + self.build_data = None + + def _check_yaml(self,): + if not os.path.exists(self.build_yaml_path.absolute()): + logger.error("%s is not exists!", self.build_yaml_path) + sys.exit(-1) + data = oebuild_util.read_yaml(yaml_path=self.build_yaml_path) + self.build_data = data + if "oebuild_env" not in data: + logger.error("%s is not valid", self.build_yaml_path.name) + sys.exit(-1) + self.oebuild_env = ParseOebuildEnvParam().parse_to_obj(data['oebuild_env']) + + def run(self): + ''' + Execute oebuild commands in order. + ''' + self._check_yaml() + + self.do_init() + + self.do_update_layer() + + self.do_build_list() + + def do_init(self, ): + ''' + Execute oebuild command : oebuild init [directory] [-u yocto_remote_url] [-b branch] + ''' + argv = [ + 'init', + ] + try: + data = oebuild_util.read_yaml(yaml_path=self.build_yaml_path) + # get openeuler repo param + if 'openeuler_layer' in data: + argv.append("-u") + argv.append(self.oebuild_env.openeuler_layer.remote_url) + + argv.append("-b") + argv.append(self.oebuild_env.openeuler_layer.version) + argv.append(self.oebuild_env.workdir) + except Exception as e_p: + raise e_p + self.app.run(argv or sys.argv[1:]) + + def do_update_layer(self, ): + ''' + Execute oebuild command : oebuild update [yocto docker layer] [-tag] + ''' + argv = [ + 'update', + 'layer' + ] + self.app.run(argv or sys.argv[1:]) + + def do_build_list(self,): + ''' + Build sequentially from the given compile parameter list. + ''' + for build_name in self.oebuild_env.build_list: + self._generate_and_bitbake(build_name=build_name) + + def _generate_and_bitbake(self, build_name): + ''' + Execute oebuild command : oebuild generate + ''' + if build_name not in self.build_data: + logger.error("lack %s dict data in %s", build_name, self.build_yaml_path) + sys.exit(-1) + compile_param_dict = self.build_data[build_name] + compile_param = ParseCompileParam().parse_to_obj(compile_param_dict=compile_param_dict) + self._generate(compile_param=compile_param, build_name=build_name) + compile_dir = os.path.join(self.oebuild_env.workdir, "build", build_name) + self._bitbake( + bitbake_cmds=compile_param.bitbake_cmds, + compile_dir=compile_dir, + build_in=compile_param.build_in) + + def _generate(self, compile_param: CompileParam, build_name): + # check compile if exists, if exists, exit with -1 + compile_dir = os.path.join(self.oebuild_env.workdir, "build", build_name) + if os.path.exists(compile_dir): + logger.error("%s has exists", compile_dir) + sys.exit(-1) + + if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: + if compile_param.docker_param is None: + logger.error("param is error, build in docker need docker_param") + sys.exit(-1) + # check src and compile_dir if exists + src_volumn_flag = False + compile_volumn_flag = False + for volumn in compile_param.docker_param.volumns: + volumn_split = volumn.split(":") + if oebuild_const.CONTAINER_SRC == volumn_split[1].strip(" "): + src_volumn_flag = True + if volumn_split[1].strip(" ").startswith(oebuild_const.CONTAINER_BUILD): + compile_volumn_flag = True + if not src_volumn_flag: + compile_param.docker_param.volumns.append( + f"{self.oebuild_env.workdir}/src:{oebuild_const.CONTAINER_SRC}" + ) + if not compile_volumn_flag: + compile_param.docker_param.volumns.append( + f"{compile_dir}:{oebuild_const.CONTAINER_BUILD}/{build_name}" + ) + compile_param_dict = ParseCompileParam().parse_to_dict(compile_param=compile_param) + os.makedirs(compile_dir) + compile_yaml_path = os.path.join(compile_dir, "compile.yaml") + oebuild_util.write_yaml(yaml_path=compile_yaml_path, data=compile_param_dict) + + def _bitbake(self, bitbake_cmds, compile_dir, build_in): + os.chdir(compile_dir) + if build_in == oebuild_const.BUILD_IN_DOCKER: + self._update_docker() + for bitbake_cmd in bitbake_cmds: + bitbake_cmd: str = bitbake_cmd.strip(" ") + argv = [ + 'bitbake', + bitbake_cmd.lstrip("bitbake") + ] + self.app.run(argv or sys.argv[1:]) + + def _update_docker(self,): + ''' + Execute oebuild command : oebuild update [yocto docker layer] [-tag] + ''' + argv = [ + 'update', + 'docker' + ] + self.app.run(argv or sys.argv[1:]) + + def main(argv=None): ''' oebuild main entrypoint @@ -197,8 +344,12 @@ def main(argv=None): colorama.init() AutoCompletion().run() - app = OebuildApp() - app.run(argv or sys.argv[1:]) + if (len(sys.argv) > 1) and 'yaml' in sys.argv[1]: + build = QuickBuild(build_yaml_path=sys.argv[1]) + build.run() + else: + app = OebuildApp() + app.run(argv or sys.argv[1:]) if __name__ == "__main__": diff --git a/src/oebuild/app/plugins/generate/generate.py b/src/oebuild/app/plugins/generate/generate.py index 1dcaf5c..1aa8b56 100644 --- a/src/oebuild/app/plugins/generate/generate.py +++ b/src/oebuild/app/plugins/generate/generate.py @@ -28,8 +28,8 @@ from ruamel.yaml.scalarstring import LiteralScalarString from oebuild.command import OebuildCommand import oebuild.util as oebuild_util from oebuild.configure import Configure -from oebuild.parse_template import BaseParseTemplate, \ - ParseTemplate, get_docker_param_dict, parse_repos_layers_local_obj +from oebuild.parse_template import BaseParseTemplate, ParseTemplate, \ + get_docker_param_dict, parse_repos_layers_local_obj from oebuild.m_log import logger, INFO_COLOR from oebuild.check_docker_tag import CheckDockerTag import oebuild.const as oebuild_const @@ -815,7 +815,10 @@ Wrong platform, please run `oebuild generate -l` to view support feature""") def get_docker_image(yocto_dir, docker_tag, configure: Configure): ''' - xxx + get docker image + first we search in yocto-meta-openeuler/.oebuild/env.yaml + second search in default ${workdir}/.oebuild/config.yaml + third get from user input ''' docker_image = oebuild_util.get_docker_image_from_yocto(yocto_dir=yocto_dir) if docker_image is None: diff --git a/src/oebuild/app/plugins/init/init.py b/src/oebuild/app/plugins/init/init.py index 29e35db..ed3944f 100644 --- a/src/oebuild/app/plugins/init/init.py +++ b/src/oebuild/app/plugins/init/init.py @@ -15,6 +15,7 @@ import shutil import os import textwrap import sys +import pathlib from oebuild.command import OebuildCommand import oebuild.util as oebuild_util @@ -138,7 +139,7 @@ please execute the follow commands next init workspace will copy config file and make new src directory ''' try: - os.mkdir(directory) + os.mkdir(pathlib.Path(directory).absolute()) except FileExistsError: return False diff --git a/src/oebuild/app/plugins/toolchain/toolchain.py b/src/oebuild/app/plugins/toolchain/toolchain.py index 613e712..2a67ae5 100644 --- a/src/oebuild/app/plugins/toolchain/toolchain.py +++ b/src/oebuild/app/plugins/toolchain/toolchain.py @@ -34,11 +34,10 @@ class Toolchain(OebuildCommand): The toolchain provides the ability to build a cross-compilation chain. ''' - help_msg = 'generate manifest from oebuild workspace' + help_msg = 'build openEuler cross-toolchain' description = textwrap.dedent('''\ - manifest provides the manifest function of generating dependent - source repositories in the build working directory, and can restore - relevant source repositories based on the manifest file + The toolchain provides similar functionality to bitbake, allowing + for the construction of an openEuler cross-toolchain. ''') def __init__(self): diff --git a/src/oebuild/app/plugins/update/update.py b/src/oebuild/app/plugins/update/update.py index 8891ba7..4f1a948 100644 --- a/src/oebuild/app/plugins/update/update.py +++ b/src/oebuild/app/plugins/update/update.py @@ -144,7 +144,7 @@ class Update(OebuildCommand): repos = None if os.path.exists(os.path.join(os.getcwd(), "compile.yaml")): compile_param_dict = oebuild_util.read_yaml(os.path.join(os.getcwd(), "compile.yaml")) - compile_param = ParseCompileParam.parse_compile_param_to_obj( + compile_param = ParseCompileParam().parse_to_obj( compile_param_dict=compile_param_dict) repos = compile_param.repos else: diff --git a/src/oebuild/bashrc.py b/src/oebuild/bashrc.py index 681089f..e25c2b4 100644 --- a/src/oebuild/bashrc.py +++ b/src/oebuild/bashrc.py @@ -33,7 +33,8 @@ class Bashrc: def set_container(self, container: Container): ''' - xxx + After setting the container parameters, all operations on bashrc + will be based on the container. ''' self.container = container self.client = DockerProxy() @@ -71,7 +72,7 @@ mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINE def get_bashrc_content(self,): ''' - xxx + get bashrc shell content ''' # return host bashrc if self.container is None: @@ -95,7 +96,7 @@ mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINE @staticmethod def restore_bashrc_content(old_content): ''' - xxx + restore bashrc content, it will delete line with oebuild_const.BASH_END_FLAG ''' new_content = '' for line in old_content.split('\n'): @@ -108,7 +109,7 @@ mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINE @staticmethod def add_bashrc(content: str, line: str): ''' - xxx + add new line to bashrc with oebuild_const.BASH_END_FLAG ''' if not content.endswith('\n'): content = content + '\n' @@ -119,9 +120,9 @@ mv /home/{oebuild_const.CONTAINER_USER}/{tmp_file} /home/{oebuild_const.CONTAINE @staticmethod def init_bashrc_content(old_content, init_command: list): ''' - xxx + add init command line to bashrc shell ''' - new_content = Bashrc.restore_bashrc_content(old_content=old_content) + new_content = Bashrc().restore_bashrc_content(old_content=old_content) for command in init_command: new_content = new_content + command + oebuild_const.BASH_END_FLAG + '\n' diff --git a/src/oebuild/local_conf.py b/src/oebuild/local_conf.py index f67f724..f612ca3 100644 --- a/src/oebuild/local_conf.py +++ b/src/oebuild/local_conf.py @@ -168,14 +168,16 @@ class LocalConf: def _add_content_to_local_conf(self, content, local_conf): user_content_flag = "#===========the content is user added==================" - if user_content_flag not in content and local_conf != "": + if user_content_flag not in content and local_conf is not None and local_conf != "": # check if exists remark sysmbol, if exists and replace it + content += f"\n{user_content_flag}\n" for line in local_conf.split('\n'): if line.startswith("#"): r_line = line.lstrip("#").strip(" ") content = content.replace(r_line, line) - content += f"\n{user_content_flag}\n" - content += local_conf + if line.strip(" ") == "None": + continue + content += line + "\n" with open(self.local_path, 'w', encoding="utf-8") as r_f: r_f.write(content) diff --git a/src/oebuild/parse_param.py b/src/oebuild/parse_param.py index f0c9606..9393ab9 100644 --- a/src/oebuild/parse_param.py +++ b/src/oebuild/parse_param.py @@ -12,11 +12,44 @@ See the Mulan PSL v2 for more details. from typing import Dict -from oebuild.struct import RepoParam, DockerParam, CompileParam, ToolchainParam +from oebuild.struct import RepoParam, DockerParam, CompileParam, ToolchainParam, OebuildEnv import oebuild.util as oebuild_util import oebuild.const as oebuild_const +class ParseOebuildEnvParam: + ''' + OebuildEnv: + workdir: str + openeuler_layer: RepoParam + build_list: Optional[list] + ''' + @staticmethod + def parse_to_obj(oebuild_env_param_dict) -> OebuildEnv: + ''' + parse dict to OebuildEnv + ''' + return OebuildEnv( + workdir=oebuild_env_param_dict['workdir'], + openeuler_layer=(None if 'openeuler_layer' not in oebuild_env_param_dict else + ParseRepoParam.parse_to_obj( + oebuild_env_param_dict['openeuler_layer'])), + build_list=(None if 'build_list' not in oebuild_env_param_dict else + oebuild_env_param_dict['build_list']) + ) + + @staticmethod + def parse_to_dict(oebuild_env_obj: OebuildEnv): + ''' + parse OebuildEnv to dict + ''' + return { + "workdir": oebuild_env_obj.workdir, + "openeuler_layer": oebuild_env_obj.openeuler_layer, + "build_list": oebuild_env_obj.build_list + } + + class ParseRepoParam: ''' RepoParam: @@ -132,12 +165,13 @@ class ParseCompileParam: repos=None if len(repos) == 0 else repos, local_conf=get_value_from_dict('local_conf', compile_param_dict, None), layers=get_value_from_dict('layers', compile_param_dict, None), - docker_param=docker_param) + docker_param=docker_param, + bitbake_cmds=get_value_from_dict('bitbake_cmds', compile_param_dict, None)) @staticmethod def parse_to_dict(compile_param: CompileParam): ''' - xxx + parse CompileParam obj to dict ''' compile_obj = {} compile_obj['build_in'] = compile_param.build_in @@ -158,7 +192,10 @@ class ParseCompileParam: if compile_param.layers is not None: compile_obj['layers'] = compile_param.layers if compile_param.build_in == oebuild_const.BUILD_IN_DOCKER: - compile_obj['docker_param'] = compile_param.docker_param + compile_obj['docker_param'] = ParseDockerParam().parse_to_dict( + compile_param.docker_param) + if compile_param.bitbake_cmds is not None: + compile_obj['bitbake_cmds'] = compile_param.bitbake_cmds return compile_obj diff --git a/src/oebuild/parse_template.py b/src/oebuild/parse_template.py index 376ee26..1ff57c8 100644 --- a/src/oebuild/parse_template.py +++ b/src/oebuild/parse_template.py @@ -164,7 +164,7 @@ class ParseTemplate: except Exception as e_p: raise e_p - def get_default_generate_compile_conf_param(self, ): + def get_default_generate_compile_conf_param(self,): ''' return default generate_compile_conf param ''' diff --git a/src/oebuild/struct.py b/src/oebuild/struct.py index 15728f8..cbd2c5a 100644 --- a/src/oebuild/struct.py +++ b/src/oebuild/struct.py @@ -30,10 +30,10 @@ class RepoParam: @dataclass class OebuildEnv: ''' - xxx + when run oebuild build.yaml, the OebuildEnv will be need in build.yaml ''' workdir: str - openeuler_layer: RepoParam + openeuler_layer: Optional[RepoParam] build_list: Optional[list] @@ -55,7 +55,7 @@ class DockerParam: @dataclass class CompileLocalParam: ''' - xxx + this is for parse to local.conf ''' sstate_mirrors: Optional[str] sstate_dir: Optional[str] @@ -65,7 +65,7 @@ class CompileLocalParam: @dataclass class CompileParamComm: ''' - xxx + this is for common param to compile.yaml ''' build_in: str machine: str @@ -79,14 +79,22 @@ class CompileParamComm: @dataclass class CompileParamHost: ''' - xxx + this is parse for host environment ''' toolchain_dir: Optional[str] nativesdk_dir: Optional[str] @dataclass -class CompileParam(CompileParamComm, CompileLocalParam, CompileParamHost): +class CompileParamBitbakeCmds: + ''' + this is for autobuild, oebuild need bitbake_cmds to run bitbake cmd automatic + ''' + bitbake_cmds: Optional[list] + + +@dataclass +class CompileParam(CompileParamComm, CompileLocalParam, CompileParamHost, CompileParamBitbakeCmds): ''' Compile is the parsed object of compile.yaml and is used to manipulate the build file ''' @@ -95,7 +103,9 @@ class CompileParam(CompileParamComm, CompileLocalParam, CompileParamHost): @dataclass class ToolchainParam: ''' - xxx + this param is for oebuld toolchain ''' + # config_list is for toolchain type,for example: config_aarch64 config_list: Optional[list] + # docker_param is for docker startup param docker_param: DockerParam diff --git a/src/oebuild/util.py b/src/oebuild/util.py index 2e9bfbe..c51fe82 100644 --- a/src/oebuild/util.py +++ b/src/oebuild/util.py @@ -284,12 +284,15 @@ def download_repo_from_manifest(repo_list, src_dir, manifest_path): ''' Download the repos set in compile.yaml based on the given base path ''' + if repo_list is None or len(repo_list) == 0: + return manifest = None if os.path.exists(manifest_path): manifest = read_yaml(pathlib.Path(manifest_path))['manifest_list'] for repo_name in repo_list: repo_dir = os.path.join(src_dir, repo_name) if repo_name in manifest: + print(repo_name) repo_obj: RepoParam = ParseRepoParam.parse_to_obj(manifest[repo_name]) repo_git = OGit(repo_dir=repo_dir, remote_url=repo_obj.remote_url, branch=None) repo_git.check_out_version(version=repo_obj.version) -- Gitee