diff --git a/README.md b/README.md index 058d94e6e4d7046db35d046a87f6d0a7da32b27f..8631fa9d1441193394e96b25f822c2a4476dcb31 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,18 @@ openEuler OpenStack SIG致力于结合多样性算力为openstack社区贡献更 - 邮件列表：openstack@openeuler.org，邮件订阅请在[页面](https://openeuler.org/zh/community/mailing-list/)中单击OpenStack链接。 - Wechat讨论群，请联系Maintainer入群 +## 本项目目录结构 + +``` +└── templet "RPM打包规范" +| └── service-templet.spec "openstack服务及对应CLI打包规范" +| └── library-templet.spec "依赖库打包规范" +| └── architecture.md "RPM分包规则" +└── tools "openstack打包、依赖分析等工作" +| └── pyporter.py "python库的RPM spec和软件包生成工作" +└── README.md "SIG文档入口" +``` + ## 项目清单 ### 统一入口 diff --git a/templet/architecture.md b/templet/architecture.md new file mode 100644 index 0000000000000000000000000000000000000000..409c88f58783a9b8dd0c79c4c13cc5fd3e0f87d6 --- /dev/null +++ b/templet/architecture.md @@ -0,0 +1,137 @@ +# 概述 + +OpenStack每个服务通常包含若干子服务，针对这些子服务，我们在打包的时候也要做拆包处理，分成若干个子RPM包。本文档规定了openEuler SIG对OpenStack服务的RPM包拆分的原则。 + +# 原则 + +## 通用原则 + +采用分层架构，RPM包结构如下图所示，以openstack-nova为例： + +``` +Level | Package | Example + | | + ┌─┐ | ┌──────────────┐ ┌────────────────────────┐ | ┌────────────────────┐ ┌────────────────────────┐ + │1│ | │ Root Package │ │ Doc Package (Optional) │ | │ openstack-nova.rpm │ │ openstack-nova-doc.rpm │ + └─┘ | └────────┬─────┘ └────────────────────────┘ | └────────────────────┘ └────────────────────────┘ + | │ | + | ┌─────────────────────┼───────────────────────────┐ | + | │ │ | | + ┌─┐ | ┌────────▼─────────┐ ┌─────────▼────────┐ | | ┌────────────────────────────┐ ┌────────────────────────┐ + │2│ | │ Service1 Package │ │ Service2 Package │ | | │ openstack-nova-compute.rpm │ │ openstack-nova-api.rpm │ + └─┘ | └────────┬─────────┘ └────────┬─────────┘ | | └────────────────────────────┘ └────────────────────────┘ + | | | | | + | └──────────┬─────────┘ | | + | | | | + ┌─┐ | ┌───────▼────────┐ | | ┌───────────────────────────┐ + │3│ | │ Common Package │ | | │ openstack-nova-common.rpm │ + └─┘ | └───────┬────────┘ | | └───────────────────────────┘ + | │ | | + | │ | | + | │ | | + ┌─┐ | ┌────────▼────────┐ ┌────────────────▼────────────────┐ | ┌──────────────────┐ ┌────────────────────────┐ + │4│ | │ Library Package ◄------------| Library Test Package (Optional) │ | │ python2-nova.rpm │ │ python2-nova-tests.rpm │ + └─┘ | └─────────────────┘ └─────────────────────────────────┘ | └──────────────────┘ └────────────────────────┘ +``` + +如图所示，分为4级 + +1. Root Package为总RPM包，原则上不包含任何文件。只做服务集合用。用户可以使用该RPM一键安装所有子RPM包。 + 如果项目有doc相关的文件，也可以单独成包（可选） +2. Service Package为子服务RPM包，包含该服务的systemd服务启动文件、自己独有的配置文件等。 +3. Common Package是共用依赖的RPM包，包含各个子服务依赖的通用配置文件、系统配置文件等。 +4. Library Package为python源码包，包含了该项目的python代码。 + 如果项目有test相关的文件，也可以单独成包（可选） + +涉及本原则的项目有： + +* openstack-nova +* openstack-cinder +* openstack-glance +* openstack-placment +* openstack-ironic + +### 特殊情况 + +有些openstack组件本身只包含一个服务，不存在子服务的概念,这种服务则只需要分为两级： + +``` + Level | Package | Example + | | + ┌─┐ | ┌──────────────┐ ┌────────────────────────┐ | ┌────────────────────────┐ ┌────────────────────────────┐ + │1│ | │ Root Package │ │ Doc Package (Optional) │ | │ openstack-keystone.rpm │ │ openstack-keystone-doc.rpm │ + └─┘ | └───────┬──────┘ └────────────────────────┘ | └────────────────────────┘ └────────────────────────────┘ + | | | + | ┌────────────┴───────────────────┐ | + ┌─┐ | ┌───────▼─────────┐ ┌────────────────▼────────────────┐ | ┌──────────────────────┐ ┌────────────────────────────┐ + │2│ | │ Library Package ◄-----| Library Test Package (Optional) │ | │ python2-keystone.rpm │ │ python2-keystone-tests.rpm │ + └─┘ | └─────────────────┘ └─────────────────────────────────┘ | └──────────────────────┘ └────────────────────────────┘ +``` + +1. Root Package RPM包包含了除python源码外的其他所有文件，包括服务启动文件、项目配置文件、系统配置文件等等。 + 如果项目有doc相关的文件，也可以单独成包（可选） +2. Library Package为python源码包，包含了该项目的python代码。 + 如果项目有test相关的文件，也可以单独成包（可选） + +涉及本原则的项目有： + +* openstack-keystone +* openstack-horizon + +还有些项目虽然有若干子RPM包，但这些子RPM包是互斥的，则这种服务的结构如下： + +``` +Level | Package | Example + | | + ┌─┐ | ┌──────────────┐ ┌────────────────────────┐ | ┌───────────────────────┐ ┌───────────────────────────┐ + │1│ | │ Root Package │ │ Doc Package (Optional) │ | │ openstack-neutron.rpm │ │ openstack-neutron-doc.rpm │ + └─┘ | └────────┬─────┘ └────────────────────────┘ | └───────────────────────┘ └───────────────────────────┘ + | │ | + | ┌─────────────────────┴───────────────────────────────┐ | + | │ | | + ┌─┐ | ┌────────▼─────────┐ ┌──────────────────┐ ┌──────────────────┐ | | ┌──────────────────────────────┐ ┌───────────────────────────────────┐ ┌───────────────────────────────────┐ + │2│ | │ Service1 Package │ │ Service2 Package │ │ Service3 Package │ | | │ openstack-neutron-server.rpm │ │ openstack-neutron-openvswitch.rpm │ │ openstack-neutron-linuxbridge.rpm │ + └─┘ | └────────┬─────────┘ └────────┬─────────┘ └────────┬─────────┘ | | └──────────────────────────────┘ └───────────────────────────────────┘ └───────────────────────────────────┘ + | | | | | | + | └────────────────────┼────────────────────┘ | | + | | | | + ┌─┐ | ┌───────▼────────┐ | | ┌──────────────────────────────┐ + │3│ | │ Common Package │ | | │ openstack-neutron-common.rpm │ + └─┘ | └───────┬────────┘ | | └──────────────────────────────┘ + | │ | | + | │ | | + | │ | | + ┌─┐ | ┌────────▼────────┐ ┌────────────────▼────────────────┐ | ┌─────────────────────┐ ┌───────────────────────────┐ + │4│ | │ Library Package ◄------| Library Test Package (Optional) │ | │ python2-neutron.rpm │ │ python2-neutron-tests.rpm │ + └─┘ | └─────────────────┘ └─────────────────────────────────┘ | └─────────────────────┘ └───────────────────────────┘ +``` + +如图所示，Service2和Service3互斥。 + +1. Root包只包含不互斥的子包，互斥的子包单独提供。 + 如果项目有doc相关的文件，也可以单独成包（可选） +2. Service Package为子服务RPM包，包含该服务的systemd服务启动文件、自己独有的配置文件等。 + 互斥的Service包不被Root包所包含，用户需要单独安装。 +3. Common Package是共用依赖的RPM包，包含各个子服务依赖的通用配置文件、系统配置文件等。 +4. Library Package为python源码包，包含了该项目的python代码。 + 如果项目有test相关的文件，也可以单独成包（可选） + +涉及本原则的项目有： + +* openstack-neutron + +## 依赖库 + +一个依赖库一般只包含一个RPM包，不需要做拆分处理。 + +``` + Level | Package | Example + | | + ┌─┐ | ┌─────────────────┐ ┌────────────────────────┐ | ┌──────────────────────────┐ ┌───────────────────────────────┐ + │1│ | │ Library Package │ │ Help Package (Optional)│ | │ python2-oslo-service.rpm │ │ python2-oslo-service-help.rpm │ + └─┘ | └─────────────────┘ └────────────────────────┘ | └──────────────────────────┘ └───────────────────────────────┘ +``` + +# 其他 + +1. openEuler社区对python2和python3 RPM包的命名有要求，python2的包前缀为*python2-*，python3的包前缀为*python3-*。因此，OpenStack要求开发者在打Library的RPM包时，也要遵守openEuler社区规范。 diff --git a/templet/library-templet.spec b/templet/library-templet.spec new file mode 100644 index 0000000000000000000000000000000000000000..b71abf51333daad79a5a1c2f94ec896fb3c1e249 --- /dev/null +++ b/templet/library-templet.spec @@ -0,0 +1,104 @@ +# --------------------------------------------------------- # +# 该模板以python-oslo.service.spec为例，删减了部分重复结构与内容 # +# ----------------------------------------------------------- # + +# openEuler社区提供了一个RPM spec自动生成的工具，叫pyporter。 +# 地址：https://gitee.com/openeuler/pyporter +# 但该工具有一些问题，比如不支持指定版本、不支持python2打包、description解析有问题等等。 +# OpenStack SIG fork了一份并进行了相应的增强和优化，放在了tools目录下，文件名叫pyporter.py +# OpenStack SIG使用该软件生成所需python依赖库的Spec文件。 + +# 常用命令： +# 1. ./pyporter -b -d +# 该命令表示从pypi上搜索对应，下载源码包，执行RPM打包动作。 +# 2. pyporter -s -d -o python-.spec +# 只下载源码包和生成spec文件，不执行打包动作 +# 例如: ./pyporter -v 2.4.0 -py2 -s -d -o python-oslo-service.spec oslo.service +# 表示下载oslo.service v2.4.0版本源码包并生成对应的python2 RPM spec + +# 下面是使用pypoter生成的python-oslo-service spec +%global _empty_manifest_terminate_build 0 +Name: python-oslo-service +Version: 2.4.0 +Release: 1 +Summary: oslo.service library +License: Apache Software License +URL: https://docs.openstack.org/oslo.service/latest/ +Source0: https://files.pythonhosted.org/packages/54/02/d889e957680e062c6302f9ab86f21f234b869d3dd831db9328b6bd63f0ba/oslo.service-2.4.0.tar.gz +BuildArch: noarch + +Requires: python2-Paste +Requires: python2-PasteDeploy +Requires: python2-Routes +Requires: python2-WebOb +Requires: python2-Yappi +Requires: python2-debtcollector +Requires: python2-eventlet +Requires: python2-fixtures +Requires: python2-greenlet +Requires: python2-oslo-concurrency +Requires: python2-oslo-config +Requires: python2-oslo-i18n +Requires: python2-oslo-log +Requires: python2-oslo-utils + +%description +# TODO:pyporter生成description有bug，需要修复 + +%package -n python2-oslo-service +Summary: oslo.service library +Provides: python-oslo-service +BuildRequires: python2-devel +BuildRequires: python2-setuptools +%description -n python2-oslo-service + + +%package help +Summary: Development documents and examples for oslo-service +Provides: python2-oslo-service-doc +%description help + + +%prep +%autosetup -n oslo-service-2.4.0 + +%build +%py2_build + +%install +%py2_install +install -d -m755 %{buildroot}/%{_pkgdocdir} +if [ -d doc ]; then cp -arf doc %{buildroot}/%{_pkgdocdir}; fi +if [ -d docs ]; then cp -arf docs %{buildroot}/%{_pkgdocdir}; fi +if [ -d example ]; then cp -arf example %{buildroot}/%{_pkgdocdir}; fi +if [ -d examples ]; then cp -arf examples %{buildroot}/%{_pkgdocdir}; fi +pushd %{buildroot} +if [ -d usr/lib ]; then + find usr/lib -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/lib64 ]; then + find usr/lib64 -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/bin ]; then + find usr/bin -type f -printf "/%h/%f\n" >> filelist.lst +fi +if [ -d usr/sbin ]; then + find usr/sbin -type f -printf "/%h/%f\n" >> filelist.lst +fi +touch doclist.lst +if [ -d usr/share/man ]; then + find usr/share/man -type f -printf "/%h/%f.gz\n" >> doclist.lst +fi +popd +mv %{buildroot}/filelist.lst . +mv %{buildroot}/doclist.lst . + +%files -n python2-oslo-service -f filelist.lst +%dir %{python2_sitelib}/* + +%files help -f doclist.lst +%{_docdir}/* + +%changelog +* Fri Apr 23 2021 Python_Bot +- Package Spec generated diff --git a/templet/service-templet.spec b/templet/service-templet.spec new file mode 100644 index 0000000000000000000000000000000000000000..a8599eb3d4a7cfc05e2ed9210cfe7376ea8500b6 --- /dev/null +++ b/templet/service-templet.spec @@ -0,0 +1,273 @@ +# ------------------------------------------------- # +# 该模板以openstack-nova为例，删减了部分重复结构与内容 # +# ------------------------------------------------- # + +# 根据需要，定义全局变量, 例如某些频繁使用的字符串、变量名等。 e.g. %global with_doc 0 +%global glocal_variable value + +# Name与gitee对应的项目保持一致 +Name: openstack-nova +# Version为openstack上游对应版本 +Version: 22.0.0 +# 初次引入时，Release为1。后续每次修改spec时，需要按数字顺序依次增长。这里的Nova spec经过了两次修改，因此为2 +# 注意，有的社区要求release的数字后面需要加?{dist}用以区分OS的版本，但openEuler社区不要求，社区的RPM构建系统会自动给RPM包名补上dist信息。 +Release: 2 +# 软件的一句话描述，Summary要简短，一般来自上游代码本身的介绍文档。 +Summary: OpenStack Compute (nova) +# 这里指定上游源码的License，openEuler社区支持的License有Apache-2.0、GPL等通用协议。 +License: Apache-2.0 +# 该项目的上游项目地址，一般指向开放的源码git仓库，如果源码地址无法提供，则可以指向项目主页 +URL: https://opendev.org/openstack/nova/ +# Source0需要指向上游软件包地址 +Source0: https://tarballs.openstack.org/nova/nova-22.0.0.tar.gz +# 新增文件按Source1、Source2...顺序依次填写 +Source1: nova-dist.conf +Source2: nova.logrotate +# 对上游源码如果需要修改，可以按顺序依次添加Git格式的Patch。例如openstack上游某些严重Bug或CVE漏洞，需要以Patch方式回合。 +Patch0001: xxx.patch +Patch0002: xxx.patch +# 软件包的目标架构，可选：x86_64、aarch64、noarch，选择不同架构会影响%install时的目录 +# BuildArch为可选项，如果不指定，则默认为与当前编译执行机架构一致。由于OpenStack是python的，因此要求指定为noarch +BuildArch: noarch + +# 编译该RPM包时需要在编译环境中提前安装的依赖，采用yum install方式安装。 +# 也可以使用yum-builddep工具进行依赖安装（yum-builddep xxx.spec） +BuildRequires: openstack-macros +BuildRequires: intltool +BuildRequires: python3-devel + +# 安装该RPM包时需要的依赖，采用yum install方式安装。 +# 注意，虽然RPM有机制自动生成python项目的依赖，但这是隐形的，不方便后期维护，因此OpenStack SIG要求开发者在spec中显示写出所有依赖项 +Requires: openstack-nova-compute = %{version}-%{release} +Requires: openstack-nova-scheduler = %{version}-%{release} + +# 该RPM包的详细描述，可以适当丰富内容，不宜过长，也不可不写。 +%description +Nova is the OpenStack project that provides a way to provision compute instances +(aka virtual servers). Nova supports creating virtual machines, baremetal servers +(through the use of ironic), and has limited support for system containers. + +# 该项目还提供多个子RPM包，一般是为了细化软件功能、文件组成，提供细粒度的RPM包。格式与上面内容相同。 +# 下面例子提供了openstack-nova-common子RPM包 +%package common +Summary: Components common to all OpenStack Nova services + +BuildRequires: systemd +BuildRequires: python3-castellan >= 0.16.0 + +Requires: python3-nova +# pre表示该依赖只在RPM包安装的%pre阶段依赖。 +Requires(pre): shadow-utils + +%description common +OpenStack Compute (Nova) +This package contains scripts, config and dependencies shared +between all the OpenStack nova services. + +# 下面例子提供了openstack-nova-compute子RPM包。 +# 注意，openstack-nova还提供了更多子RPM包，例如api、scheduler、conductor等。本示例做了删减。 +# +# 子RPM包的命名规则： +# 以项目名为缺省值，使用"项目名-子服务名"的方式，例如这里的openstack-nova-compute +# openstack-nova为服务名，compute为子服务名。 +%package compute +Summary: OpenStack Nova Compute Service + +Requires: openstack-nova-common = %{version}-%{release} +Requires: curl +Requires(pre): qemu + +%description compute +This package contains scripts, config for nova-compute service + +# 下面例子提供了python-nova子RPM包。 +%package -n python-nova +Summary: Nova Python libraries + +Requires: openssl +Requires: openssh +Requires: sudo +Requires: python-paramiko + +# -n表示非缺省，包的全名为python-nova，而不是openstack-nova-python-nova +%description -n python-nova +OpenStack Compute (Nova) +This package contains the nova Python library. + +# 下面例子提供了openstack-nova-doc子RPM包。 +# 目前doc和test类的子RPM不做强制要求，可以不提供。 +%package doc +Summary: Documentation for OpenStack Compute +BuildRequires: graphviz +BuildRequires: python3-openstackdocstheme + +%description doc +OpenStack Compute (Nova) +This package contains documentation files for nova. + +# 下面进入RPM包正式编译流程，按照%prep、%build、%install、%check、%clean的步骤进行。 +# %prep: 打包准备阶段执行一些命令（如，解压源码包，打补丁等），以便开始编译。一般仅包含 "%autosetup"；如果源码包需要解压并切换至 NAME 目录，则输入 "%autosetup -n NAME"。 +# %build: 包含构建阶段执行的命令，构建完成后便开始后续安装。程序应该包含有如何编译的介绍。 +# %install: 包含安装阶段执行的命令。命令将文件从 %{_builddir} 目录安装至 %{buildroot} 目录。 +# %check: 包含测试阶段执行的命令。此阶段在 %install 之后执行，通常包含 "make test" 或 "make check" 命令。此阶段要与 %build 分开，以便在需要时忽略测试。 +# %clean: 可选步骤，清理安装目录的命令。一般只包含：rm -rf %{buildroot} + +# nova的prep阶段进行源码包解压、删除无用文件。 +%prep +%autosetup -n nova-%{upstream_version} -S git +find . \( -name .gitignore -o -name .placeholder \) -delete +find nova -name \*.py -exec sed -i '/\/usr\/bin\/env python/{d;q}' {} + +%py_req_cleanup + +# build阶段使用py3_build命令快速编译，并生成、修改个别文件（如nova.conf配置文件） +%build +PYTHONPATH=. oslo-config-generator --config-file=etc/nova/nova-config-generator.conf +PYTHONPATH=. oslopolicy-sample-generator --config-file=etc/nova/nova-policy-generator.conf +%{py3_build} +%{__python3} setup.py compile_catalog -d build/lib/nova/locale -D nova +sed -i 's|group/name|group;name|; s|\[DEFAULT\]/|DEFAULT;|' etc/nova/nova.conf.sample + +# install阶段使用py3_install命令快速安装，配置文件对应权限、生成doc、systemd服务启动文件等。 +%install +%{py3_install} +export PYTHONPATH=. +sphinx-build -b html doc/source doc/build/html +rm -rf doc/build/html/.{doctrees,buildinfo} +install -p -D -m 644 doc/build/man/*.1 %{buildroot}%{_mandir}/man1/ +install -d -m 755 %{buildroot}%{_sharedstatedir}/nova +install -d -m 755 %{buildroot}%{_sharedstatedir}/nova/buckets +install -d -m 755 %{buildroot}%{_sharedstatedir}/nova/instances +cat > %{buildroot}%{_sysconfdir}/nova/release < os_xenapi/client.py </dev/null || groupadd -r nova --gid 162 +if ! getent passwd nova >/dev/null; then + useradd -u 162 -r -g nova -G nova,nobody -d %{_sharedstatedir}/nova -s /sbin/nologin -c "OpenStack Nova Daemons" nova +fi +exit 0 + +# openstack-nova-compute类似 +%pre compute +usermod -a -G qemu nova +usermod -a -G libvirt nova + +# openstack-nova-compute安装之后会刷新systemd配置。 +%post compute +%systemd_post %{name}-compute.service + +# openstack-nova-compute卸载之前会刷新systemd配置。 +%preun compute +%systemd_preun %{name}-compute.service + +# openstack-nova-compute卸载之后会做类似操作。 +%postun compute +%systemd_postun_with_restart %{name}-compute.service + +# spec的最后是%files部分, 此部分列出了需要被打包的文件和目录，即RPM包里包含哪些文件和目录，以及这些文件和目录会被安装到哪里。 + +# %files表示openstack-nova RPM包含的东西，这些什么都没写，表示openstack-nova RPM包是个空包, 但是rpm本身对于其他rpm有依赖。 +%files +# 下面表示openstack-nova-common RPM包里包含的文件和目录，有nova.conf配置文件、policy.json权限文件、一些nova可执行文件等等。 +%files common -f nova.lang +%license LICENSE +%doc etc/nova/policy.yaml.sample +%dir %{_datarootdir}/nova +%attr(-, root, nova) %{_datarootdir}/nova/nova-dist.conf +%{_datarootdir}/nova/interfaces.template +%dir %{_sysconfdir}/nova +%{_sysconfdir}/nova/release +%config(noreplace) %attr(-, root, nova) %{_sysconfdir}/nova/nova.conf +%config(noreplace) %attr(-, root, nova) %{_sysconfdir}/nova/api-paste.ini +%config(noreplace) %attr(-, root, nova) %{_sysconfdir}/nova/rootwrap.conf +%config(noreplace) %attr(-, root, nova) %{_sysconfdir}/nova/policy.json +%config(noreplace) %{_sysconfdir}/logrotate.d/openstack-nova +%config(noreplace) %{_sysconfdir}/sudoers.d/nova +%dir %attr(0750, nova, root) %{_localstatedir}/log/nova +%dir %attr(0755, nova, root) %{_localstatedir}/run/nova +%{_bindir}/nova-manage +%{_bindir}/nova-policy +%{_bindir}/nova-rootwrap +%{_bindir}/nova-rootwrap-daemon +%{_bindir}/nova-status +%if 0%{?with_doc} +%{_mandir}/man1/nova*.1.gz +%endif +%defattr(-, nova, nova, -) +%dir %{_sharedstatedir}/nova +%dir %{_sharedstatedir}/nova/buckets +%dir %{_sharedstatedir}/nova/instances +%dir %{_sharedstatedir}/nova/keys +%dir %{_sharedstatedir}/nova/networks +%dir %{_sharedstatedir}/nova/tmp + +# openstack-nova-compute RPM包包含nova-compute可执行文件、systemd服务配置文件、nova-compute提权文件。 +# 会分别安装到/usr/bin等系统目录。 +%files compute +%{_bindir}/nova-compute +%{_unitdir}/openstack-nova-compute.service +%{_datarootdir}/nova/rootwrap/compute.filters + +# python-nova RPM包里包含了nova的python文件，会被安装到python3_sitelib目录中，一般是/usr/lib/python3/site-packages/ +%files -n python-nova +%license LICENSE +%{python3_sitelib}/nova +%{python3_sitelib}/nova-*.egg-info +%exclude %{python3_sitelib}/nova/tests + +# openstack-nova-doc RPM包里只有doc的相关文件 +%files doc +%license LICENSE +%doc doc/build/html + +# 最后别忘了填写changelog，每次修改spec文件，都要新增changelog信息，从上到下按时间由近及远的顺序。 +%changelog +* Sat Feb 20 2021 wangxiyuan +- Fix require issue + +* Fri Jan 15 2021 joec88 +- openEuler build version + + +# 附录： +# 上面spec代码中使用了大量的RPM宏定义，更多相关宏的详细说明参考fedora官方wiki +# https://fedoraproject.org/wiki/How_to_create_an_RPM_package/zh-cn#.E5.AE.8F diff --git a/tools/pyporter.py b/tools/pyporter.py new file mode 100644 index 0000000000000000000000000000000000000000..cc0f4600ad30fa2483ba00b6a6b1104bf04229ca --- /dev/null +++ b/tools/pyporter.py @@ -0,0 +1,633 @@ +#!/usr/bin/python3 +""" +This is a packager bot for python modules from pypi.org +""" +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: Shinwell_Hu Myeuler +# Create: 2020-05-07 +# Description: provide a tool to package python module automatically +# +# ****************************************************************************** +# +# OpenStack SIG fork version +# Update: 2021-04-23 +# +# How to use: +# 1. ./pyporter -d -s -o python-xxx.spec xxx +# download xxx package(newest version) from pypi and create RPM spec file named python-xxx.spec +# 2. ./pyporter -d -b xxx +# download xxx package(newest version) from pypi and build RPM package automaticly(python3 by default) +# 3. ./pyporter -v 1.0.0 -d -s -o python-xxx.spec xxx +# download xxx package(version 1.0.0) from pypi and create RPM spec file named python-xxx.spec +# 4. ./pyporter -v 1.0.0 -py2 -d -b xxx +# download xxx package(version 1.0.0) from pypi and build python2 RPM package automaticly +# +# Supported parameter： +# "-s" | "--spec" | Create spec file +# "-R" | "--requires" | Get required python modules +# "-b" | "--build" | Build rpm package +# "-B" | "--buildinstall" | Build&Install rpm package +# "-r" | "--rootpath" | Build rpm package in root path +# "-d" | "--download" | Download source file indicated path +# "-p" | "--path" | indicated path to store files +# "-j" | "--json" | Get Package JSON info +# "-o" | "--output" | Output to file +# "-t" | "--type" | Build module type : python, perl... +# "-a" | "--arch" | Build module with arch +# "-v" | "--version" | Package version +# "-py2" | "--python2" | Build python2 package +# ****************************************************************************** + +import urllib +import urllib.request +from pprint import pprint +from os import path +import json +import sys +import re +import datetime +import argparse +import subprocess +import os +import platform +from pathlib import Path +import hashlib + +# python3-wget is not default available on openEuler yet. +# import wget + + +json_file_template = '{pkg_name}.json' +name_tag_template = 'Name:\t\t{pkg_name}' +summary_tag_template = 'Summary:\t{pkg_sum}' +version_tag_template = 'Version:\t{pkg_ver}' +release_tag_template = 'Release:\t1' +license_tag_template = 'License:\t{pkg_lic}' +home_tag_template = 'URL:\t\t{pkg_home}' +source_tag_template = 'Source0:\t{pkg_source}' + +buildreq_tag_template = 'BuildRequires:\t{req}' + + +# TODO List +# 1. Need a reliable way to get description of module .. Partially done +# 2. requires_dist has some dependency restirction, need to present +# 3. dependency outside python (i.e. pycurl depends on libcurl) doesn't exist in pipy + + +class PyPorter: + __url_template_latest = 'https://pypi.org/pypi/{pkg_name}/json' + __url_template_version = 'https://pypi.org/pypi/{pkg_name}/{pkg_version}/json' + __build_noarch = True + __json = None + __module_name = "" + __spec_name = "" + __pkg_name = "" + __pkg_version = "" + __is_python2 = False + + def __init__(self, arch, pkg, version, is_python2): + """ + receive json from pypi.org + """ + if version: + url = self.__url_template_version.format(pkg_name=pkg, pkg_version=version) + else: + url = self.__url_template_latest.format(pkg_name=pkg) + resp = "" + with urllib.request.urlopen(url) as u: + self.__json = json.loads(u.read().decode('utf-8')) + if self.__json is not None: + self.__module_name = self.__json["info"]["name"] + self.__module_name = self.__module_name.replace(".", "-") + self.__spec_name = "python-" + self.__module_name + if is_python2: + self.__is_python2 = True + self.__pkg_name = "python2-" + self.__module_name + else: + self.__is_python2 = False + self.__pkg_name = "python3-" + self.__module_name + self.__build_noarch = self.__get_buildarch() + + if arch: + self.__build_noarch = False + + def get_spec_name(self): + return self.__spec_name + + def get_module_name(self): + return self.__module_name + + def get_pkg_name(self): + return self.__pkg_name + + def get_version(self): + return self.__json["info"]["version"] + + def get_summary(self): + return self.__json["info"]["summary"] + + def get_home(self): + return self.__json["info"]["project_urls"]["Homepage"] + + def get_license(self): + """ + By default, the license info can be achieved from json["info"]["license"] + in rare cases it doesn't work. + We fall back to json["info"]["classifiers"], it looks like License :: OSI Approved :: BSD Clause + """ + if self.__json["info"]["license"] != "": + return self.__json["info"]["license"] + for k in self.__json["info"]["classifiers"]: + if k.startswith("License"): + ks = k.split("::") + return ks[2].strip() + return "" + + def get_source_info(self): + """ + return a map of source filename, md5 of source, source url + return None in errors + """ + v = self.__json["info"]["version"] + rs = self.__json["releases"][v] + for r in rs: + if r["packagetype"] == "sdist": + return {"filename": r["filename"], "md5": r["md5_digest"], "url": r["url"]} + return None + + def get_source_url(self): + """ + return URL for source file for the latest version + return "" in errors + """ + s_info = self.get_source_info() + if s_info: + return s_info.get("url") + return "" + + def get_requires(self): + """ + return all requires no matter if extra is required. + """ + rs = self.__json["info"]["requires_dist"] + if rs is None: + return + for r in rs: + idx = r.find(";") + if idx != -1: + r = r[:idx] + print("Requires:\t" + transform_module_name(r, self.__is_python2)) + + def __get_buildarch(self): + """ + if this module has a prebuild package for amd64, then it is arch dependent. + print BuildArch tag if needed. + """ + v = self.__json["info"]["version"] + rs = self.__json["releases"][v] + for r in rs: + if r["packagetype"] == "bdist_wheel": + if r["url"].find("amd64") != -1: + return False + return True + + def is_build_noarch(self): + return self.__build_noarch + + def get_buildarch(self): + if (self.__build_noarch == True): + print("BuildArch:\tnoarch") + + def get_description(self): + """ + return description. + Usually it's json["info"]["description"] + If it's rst style, then only use the content for the first paragraph, and remove all tag line. + For empty description, use summary instead. + """ + desc = self.__json["info"]["description"].splitlines() + res = [] + paragraph = 0 + for d in desc: + if len(d.strip()) == 0: + continue + first_char = d.strip()[0] + ignore_line = False + if d.strip().startswith("===") or d.strip().startswith("---"): + paragraph = paragraph + 1 + ignore_line = True + elif d.strip().startswith(":") or d.strip().startswith(".."): + ignore_line = True + if ignore_line != True and paragraph == 1: + res.append(d) + if paragraph >= 2: + del res[-1] + return "\n".join(res) + if res: + return "\n".join(res) + elif paragraph == 0: + return self.__json["info"]["description"] + else: + return self.__json["info"]["summary"] + + def get_build_requires(self): + req_list = [] + rds = self.__json["info"]["requires_dist"] + if rds is not None: + for rp in rds: + br = refine_requires(rp, self.__is_python2) + if br == "": + continue + # + # Do not output BuildRequires: + # just collect all build requires and using pip to install + # than can help to build all rpm withoud trap into + # build dependency nightmare + # + # print(buildreq_tag_template.format(req=br)) + name = str.lstrip(br).split(" ") + req_list.append(name[0]) + return req_list + + def prepare_build_requires(self): + if self.__is_python2: + print(buildreq_tag_template.format(req='python2-devel')) + print(buildreq_tag_template.format(req='python2-setuptools')) + else: + print(buildreq_tag_template.format(req='python3-devel')) + print(buildreq_tag_template.format(req='python3-setuptools')) + if not self.__build_noarch: + if self.__is_python2: + print(buildreq_tag_template.format(req='python2-cffi')) + else: + print(buildreq_tag_template.format(req='python3-cffi')) + print(buildreq_tag_template.format(req='gcc')) + print(buildreq_tag_template.format(req='gdb')) + + def prepare_pkg_build(self): + if self.__is_python2: + print("%py2_build") + else: + print("%py3_build") + + def prepare_pkg_install(self): + if self.__is_python2: + print("%py2_install") + else: + print("%py3_install") + + def prepare_pkg_files(self): + if self.__build_noarch: + if self.__is_python2: + print("%dir %{python2_sitelib}/*") + else: + print("%dir %{python3_sitelib}/*") + else: + if self.__is_python2: + print("%dir %{python2_sitearch}/*") + else: + print("%dir %{python3_sitearch}/*") + + def store_json(self, spath): + """ + save json file + """ + fname = json_file_template.format(pkg_name=self.__pkg_name) + json_file = os.path.join(spath, fname) + + # if file exist, do nothing + if path.exists(json_file) and path.isfile(json_file): + with open(json_file, 'r') as f: + resp = json.load(f) + else: + with open(json_file, 'w') as f: + json.dump(self.__json, f) + + +def transform_module_name(n, is_python2): + """ + return module name with version restriction. + Any string with '.' or '/' is considered file, and will be ignored + Modules start with python- will be changed to python3- for consistency. + """ + # remove () + prefix = "python2-" if is_python2 else "python3-" + module_name = n.split(' ', 1)[0].split("[")[0] + if module_name.startswith("python-"): + module_name = module_name.replace("python-", prefix) + else: + module_name = prefix + module_name + module_name = module_name.replace(".", "-") + return module_name + + +def refine_requires(req, is_python2): + """ + return only requires without ';' (thus no extra) + """ + ra = req.split(";", 1) + # + # Do not add requires which has ;, which is often has very complicated precondition + # TODO: need more parsing of the denpency after ; + return transform_module_name(ra[0], is_python2) + + +def download_source(porter, tgtpath): + """ + download source file from url, and save it to target path + """ + if not os.path.exists(tgtpath): + print("download path %s does not exist\n", tgtpath) + return False + s_info = porter.get_source_info() + if s_info is None: + print("analyze source info error") + return False + s_url = s_info.get("url") + s_path = os.path.join(tgtpath, s_info.get("filename")) + if os.path.exists(s_path): + with open(s_path, 'rb') as f: + md5obj = hashlib.md5() + md5obj.update(f.read()) + _hash = str(md5obj.hexdigest()).lower() + if s_info.get("md5") == _hash: + print("same source file exists, skip") + return True + return subprocess.call(["wget", s_url, "-P", tgtpath]) + + +def prepare_rpm_build_env(root): + """ + prepare environment for rpmbuild + """ + if not os.path.exists(root): + print("Root path %s does not exist\n" & buildroot) + return "" + + buildroot = os.path.join(root, "rpmbuild") + if not os.path.exists(buildroot): + os.mkdir(buildroot) + + for sdir in ['SPECS', 'BUILD', 'SOURCES', 'SRPMS', 'RPMS', 'BUILDROOT']: + bpath = os.path.join(buildroot, sdir) + if not os.path.exists(bpath): + os.mkdir(bpath) + + return buildroot + + +def try_pip_install_package(pkg): + """ + install packages listed in build requires + """ + # try pip installation + pip_name = pkg.split("-") + if len(pip_name) == 2: + ret = subprocess.call(["pip3", "install", "--user", pip_name[1]]) + else: + ret = subprocess.call(["pip3", "install", "--user", pip_name[0]]) + + if ret != 0: + print("%s can not be installed correctly, Fix it later, go ahead to do building..." % pip_name) + + # + # TODO: try to build anyway, fix it later + # + return True + + +def package_installed(pkg): + print(pkg) + ret = subprocess.call(["rpm", "-qi", pkg]) + if ret == 0: + return True + + return False + + +def dependencies_ready(req_list): + """ + TODO: do not need to do dependency check here, do it in pyporter_run + """ + # if (try_pip_install_package(req) == False): + # return req + return "" + + +def build_package(specfile): + """ + build rpm package with rpmbuild + """ + ret = subprocess.call(["rpmbuild", "-ba", specfile]) + return ret + + +def build_install_rpm(porter, rootpath): + ret = build_rpm(porter, rootpath) + if ret != "": + return ret + + arch = "noarch" + if porter.is_build_noarch(): + arch = "noarch" + else: + arch = platform.machine() + + pkgname = os.path.join(rootpath, "rpmbuild", "RPMS", arch, porter.get_pkg_name() + "*") + ret = subprocess.call(["rpm", "-ivh", pkgname]) + if ret != 0: + return "Install failed\n" + + return "" + + +def build_rpm(porter, rootpath): + """ + full process to build rpm + """ + buildroot = prepare_rpm_build_env(rootpath) + if buildroot == "": + return False + + specfile = os.path.join(buildroot, "SPECS", porter.get_spec_name() + ".spec") + + req_list = build_spec(porter, specfile) + ret = dependencies_ready(req_list) + if ret != "": + print("%s can not be installed automatically, Please handle it" % ret) + return ret + + download_source(porter, os.path.join(buildroot, "SOURCES")) + + build_package(specfile) + + return "" + + +def build_spec(porter, output): + """ + print out the spec file + """ + if os.path.isdir(output): + output = os.path.join(output, porter.get_spec_name() + ".spec") + tmp = sys.stdout + if output != "": + sys.stdout = open(output, 'w+') + + print("%global _empty_manifest_terminate_build 0") + print(name_tag_template.format(pkg_name=porter.get_spec_name())) + print(version_tag_template.format(pkg_ver=porter.get_version())) + print(release_tag_template) + print(summary_tag_template.format(pkg_sum=porter.get_summary())) + print(license_tag_template.format(pkg_lic=porter.get_license())) + print(home_tag_template.format(pkg_home=porter.get_home())) + print(source_tag_template.format(pkg_source=porter.get_source_url())) + porter.get_buildarch() + print("") + porter.get_requires() + print("") + print("%description") + print(porter.get_description()) + print("") + + print("%package -n {name}".format(name=porter.get_pkg_name())) + print(summary_tag_template.format(pkg_sum=porter.get_summary())) + print("Provides:\t" + porter.get_spec_name()) + + porter.prepare_build_requires() + + build_req_list = porter.get_build_requires() + + print("%description -n " + porter.get_pkg_name()) + print(porter.get_description()) + print("") + print("%package help") + print("Summary:\tDevelopment documents and examples for {name}".format(name=porter.get_module_name())) + print("Provides:\t{name}-doc".format(name=porter.get_pkg_name())) + print("%description help") + print(porter.get_description()) + print("") + print("%prep") + print("%autosetup -n {name}-{ver}".format(name=porter.get_module_name(), ver=porter.get_version())) + print("") + print("%build") + porter.prepare_pkg_build() + print("") + print("%install") + porter.prepare_pkg_install() + print("install -d -m755 %{buildroot}/%{_pkgdocdir}") + print("if [ -d doc ]; then cp -arf doc %{buildroot}/%{_pkgdocdir}; fi") + print("if [ -d docs ]; then cp -arf docs %{buildroot}/%{_pkgdocdir}; fi") + print("if [ -d example ]; then cp -arf example %{buildroot}/%{_pkgdocdir}; fi") + print("if [ -d examples ]; then cp -arf examples %{buildroot}/%{_pkgdocdir}; fi") + print("pushd %{buildroot}") + print("if [ -d usr/lib ]; then") + print("\tfind usr/lib -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("if [ -d usr/lib64 ]; then") + print("\tfind usr/lib64 -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("if [ -d usr/bin ]; then") + print("\tfind usr/bin -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("if [ -d usr/sbin ]; then") + print("\tfind usr/sbin -type f -printf \"/%h/%f\\n\" >> filelist.lst") + print("fi") + print("touch doclist.lst") + print("if [ -d usr/share/man ]; then") + print("\tfind usr/share/man -type f -printf \"/%h/%f.gz\\n\" >> doclist.lst") + print("fi") + print("popd") + print("mv %{buildroot}/filelist.lst .") + print("mv %{buildroot}/doclist.lst .") + print("") + print("%files -n {name} -f filelist.lst".format(name=porter.get_pkg_name())) + + porter.prepare_pkg_files() + + print("") + print("%files help -f doclist.lst") + print("%{_docdir}/*") + print("") + print("%changelog") + date_str = datetime.date.today().strftime("%a %b %d %Y") + print("* {today} Python_Bot ".format(today=date_str)) + print("- Package Spec generated") + + sys.stdout = tmp + + return build_req_list + + +def do_args(root): + parser = argparse.ArgumentParser() + + parser.add_argument("-s", "--spec", help="Create spec file", action="store_true") + parser.add_argument("-R", "--requires", help="Get required python modules", action="store_true") + parser.add_argument("-b", "--build", help="Build rpm package", action="store_true") + parser.add_argument("-B", "--buildinstall", help="Build&Install rpm package", action="store_true") + parser.add_argument("-r", "--rootpath", help="Build rpm package in root path", type=str, default=dft_root_path) + parser.add_argument("-d", "--download", help="Download source file indicated path", action="store_true") + parser.add_argument("-p", "--path", help="indicated path to store files", type=str, default=os.getcwd()) + parser.add_argument("-j", "--json", help="Get Package JSON info", action="store_true") + parser.add_argument("-o", "--output", help="Output to file", type=str, default="") + parser.add_argument("-t", "--type", help="Build module type : python, perl...", type=str, default="python") + parser.add_argument("-a", "--arch", help="Build module with arch", action="store_true") + parser.add_argument("-v", "--version", help="Package version", type=str, default="") + parser.add_argument("-py2", "--python2", help="Build python2 package", action="store_true") + parser.add_argument("pkg", type=str, help="The Python Module Name") + + return parser + + +def porter_creator(t_str, arch, pkg, version, is_python2): + if t_str == "python": + return PyPorter(arch, pkg, version, is_python2) + + return None + + +if __name__ == "__main__": + + dft_root_path = os.path.join(str(Path.home())) + + parser = do_args(dft_root_path) + + args = parser.parse_args() + + porter = porter_creator(args.type, args.arch, args.pkg, args.version, args.python2) + if porter is None: + print("Type %s is not supported now\n" % args.type) + sys.exit(1) + + if args.requires: + req_list = porter.get_build_requires() + if req_list is not None: + for req in req_list: + print(req) + elif args.spec: + build_spec(porter, args.output) + elif args.build: + ret = build_rpm(porter, args.rootpath) + if ret != "": + print("build failed : BuildRequire : %s\n" % ret) + sys.exit(1) + elif args.buildinstall: + ret = build_install_rpm(porter, args.rootpath) + if ret != "": + print("Build & install failed\n") + sys.exit(1) + elif args.download: + download_source(porter, args.path) + elif args.json: + porter.store_json(args.path)