diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml new file mode 100644 index 0000000000000000000000000000000000000000..2a5d87940f125ebcc7b26fc00397f473f13bcb7a --- /dev/null +++ b/APPLICATION/spark/buildspec.yml @@ -0,0 +1,57 @@ +name: &NAME +version: &VERSION +image_type: &IMAGE_TYPE +baseos_version: &BASEOS_VERSION + +# 定义镜像仓库信息 +repository_info: + acr_alinux: &ACR_ALINUX alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3 + acr_anolis: &ACR_ANOLIS anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis + +t-one: + workspace: &WORKSPACE container_ci_test + project: &PROJECT default_container_ci_test + test_suite: &TEST_SUITE image-ci-test + test_conf: &TEST_CONF group=big_data_container + test_case: &TEST_CASE big_data_container_default + cloud_server_tag: &CLOUD_SERVER_TAG [anolis-container-ci-x86, anolis-container-ci-arm] + +images: + BuildSparkECSApplicationDockerImage: + build: true + docker_file: + path: APPLICATION/spark/general/Dockerfile + variable: + # - &SPARK33_JAVA8_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.6")] + # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] + # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] + # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] + - &SPARK33_JAVA8_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "3")] + - &SPARK33_JAVA11_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "3")] + - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] + - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] + scene: + args: [ + # *SPARK33_JAVA8_A86, *SPARK33_JAVA11_A86, + # *SPARK33_JAVA8_A88, *SPARK33_JAVA11_A88, + *SPARK33_JAVA8_A3, *SPARK33_JAVA11_A3, + *SPARK33_JAVA8_A23, *SPARK33_JAVA11_A23] + tags: + # - ["3.3.3-java8-8.6", "3.3-java8-8.6", "3-java8-8.6"] + # - ["3.3.3-java11-8.6", "3.3-java11-8.6", "3-java11-8.6", "3.3.3-8.6", "3.3-8.6", "3-8.6"] + # - ["3.3.3-java8-8.8", "3.3-java8-8.8", "3-java8-8.8"] + # - ["3.3.3-java11-8.8", "3.3-java11-8.8", "3-java11-8.8", "3.3.3-8.8", "3.3-8.8", "3-8.8"] + - ["3.3.3-java8-3", "3.3-java8-3", "3-java8-3"] + - ["3.3.3-java11-3", "3.3-java11-3", "3-java11-3", "3.3.3-3", "3.3-3", "3-3"] + - ["3.3.3-java8-23", "3.3-java8-23", "3-java8-23", "3.3.3-java8", "3.3-java8", "3-java8"] + - ["3.3.3-java11-23", "3.3-java11-23", "3-java11-23", "3.3.3-23", "3.3-23", "3-23", "3.3.3", "3.3", "3", "latest"] + registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] + test_config: + # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..b5152b080f1a46f4559379404d6c44fa6f1022bd --- /dev/null +++ b/APPLICATION/spark/general/Dockerfile @@ -0,0 +1,89 @@ +ARG SPARK_VERSION +ARG BASE_OS + +FROM registry.openanolis.cn/openanolis/anolisos:23 as downloader +ARG SPARK_VERSION + +ARG SPARK_FILE=spark-${SPARK_VERSION}-bin-hadoop3 + +RUN yum install -y tar gzip + +RUN curl -O -m 600 https://mirrors.ustc.edu.cn/apache/spark/spark-${SPARK_VERSION}/${SPARK_FILE}.tgz +RUN gzip -d ${SPARK_FILE}.tgz +RUN tar xf ${SPARK_FILE}.tar +RUN mv ${SPARK_FILE} spark + +COPY tini-amd64 / +COPY tini-arm64 / + +RUN set -eux; \ + if [ "$(uname -m)" == "aarch64" ]; then \ + mv tini-arm64 tini; \ + else \ + mv tini-amd64 tini; \ + fi; \ + chmod +x tini + +FROM ${BASE_OS} + +ARG SPARK_VERSION +ARG OS_SUFFIX +ARG JAVA_VERSION +ARG spark_uid=185 + +LABEL org.opencontainers.image.title="Spark" \ + org.opencontainers.image.version="${SPARK_VERSION}-${OS_SUFFIX}" + +ENV SPARK_VERSION ${SPARK_VERSION} +ENV JAVA_HOME /etc/alternatives/jre +ENV PATH $JAVA_HOME/bin:$PATH + +RUN set -eux; \ + yum install -y net-tools iputils hostname; \ + case "${JAVA_VERSION}" in \ + "8" ) \ + yum install -y java-1.8.0-alibaba-dragonwell \ + ;; \ + "11" ) \ + yum install -y java-11-alibaba-dragonwell \ + ;; \ + * ) \ + echo unsupported java version: ${JAVA_VERSION}; exit 1 \ + ;; \ + esac; \ + # (java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}' | xargs -I{} ln -s {} ${JAVA_HOME}); \ + mkdir -p /opt/spark; \ + mkdir -p /opt/spark/examples; \ + mkdir -p /opt/spark/work-dir; \ + touch /opt/spark/RELEASE; \ + echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su; \ + chgrp root /etc/passwd && chmod ug+rw /etc/passwd; \ + yum clean all; \ + groupadd --system --gid=${spark_uid} spark; \ + useradd --system --uid=${spark_uid} --gid=spark spark; \ + chown -R spark:spark /opt/spark + +COPY --from=downloader --chown=spark:spark /spark/jars /opt/spark/jars +COPY --from=downloader --chown=spark:spark /spark/bin /opt/spark/bin +COPY --from=downloader --chown=spark:spark /spark/sbin /opt/spark/sbin +COPY --from=downloader --chown=spark:spark /spark/kubernetes/dockerfiles/spark/entrypoint.sh /opt/ +COPY --from=downloader --chown=spark:spark /spark/kubernetes/dockerfiles/spark/decom.sh /opt/ +COPY --from=downloader --chown=spark:spark /spark/examples /opt/spark/examples +COPY --from=downloader --chown=spark:spark /spark/kubernetes/tests /opt/spark/tests +COPY --from=downloader --chown=spark:spark /spark/data /opt/spark/data + +ENV SPARK_HOME /opt/spark +ENV PATH ${SPARK_HOME}/bin:${PATH} + +WORKDIR /opt/spark/work-dir + +RUN set -eux; \ + chmod g+w /opt/spark/work-dir; \ + chmod a+x /opt/decom.sh + +COPY --from=downloader /tini /usr/bin/tini + +ENTRYPOINT [ "/opt/entrypoint.sh" ] + +# Specify the User that the actual main process will run as +USER ${spark_uid} diff --git a/APPLICATION/spark/general/tini-amd64 b/APPLICATION/spark/general/tini-amd64 new file mode 100755 index 0000000000000000000000000000000000000000..86cb766c6b8af2564ac9201df913fe079acae3d1 Binary files /dev/null and b/APPLICATION/spark/general/tini-amd64 differ diff --git a/APPLICATION/spark/general/tini-arm64 b/APPLICATION/spark/general/tini-arm64 new file mode 100755 index 0000000000000000000000000000000000000000..8f8423253c0ee96090659616d5f05384bafacd76 Binary files /dev/null and b/APPLICATION/spark/general/tini-arm64 differ diff --git a/APPLICATION/spark/version.yml b/APPLICATION/spark/version.yml new file mode 100644 index 0000000000000000000000000000000000000000..974d70c7a5fcdd990afbd043105729ef96119906 --- /dev/null +++ b/APPLICATION/spark/version.yml @@ -0,0 +1,8 @@ +# 版本关系依赖表,默认继承 version-base.yml 配置,可重写覆盖 +BaseDependency: ../version-base.yml +Dependency: + name: spark + image_type: application + versions: + general: + baseos_version: [[ignored]]