diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d66cd217ea7fde01fd19c138c3c9be8684a38666 --- /dev/null +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -0,0 +1,125 @@ +FROM anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/cuda:12.1.1-23-devel + +RUN yum install -y python3-pip +RUN cp -r /etc/yum.repos.d/ /etc/yum.repos.d.bak && cp /etc/pip.conf /etc/pip.conf.bak + +RUN sed -i 's#https://mirrors.openanolis.cn#http://mirrors.cloud.aliyuncs.com#g' /etc/yum.repos.d/*.repo +COPY pip.conf /etc/pip.conf + +RUN yum install -y \ + automake \ + libtool \ + system-rpm-config \ + wget \ + git git-lfs \ + blas \ + glibc-locale-source \ + libSM \ + langpacks-zh_CN \ + gcc gcc-c++ && yum clean all + +RUN mkdir -p /var/modelscope +COPY requirements /var/modelscope + +RUN yum install -y jupyterlab && yum clean all +RUN yum install -y python3-modelscope \ + && yum clean all + +RUN yum install -y 'python3dist(sacremoses)' \ + 'python3dist(pycryptodomex)' \ + 'python3dist(easydict)' \ + 'python3dist(pywavelets)' \ + 'python3dist(virtualenv)' \ + 'python3dist(unicodedata2)' \ + 'python3dist(munkres)' \ + 'python3dist(jamo)' \ + 'python3dist(flatbuffers)' \ + 'python3dist(distlib)' \ + 'python3dist(cymem)' \ + 'python3dist(widgetsnbextension)' \ + 'python3dist(wasabi)' \ + 'python3dist(virtualenv)' \ + 'python3dist(unidecode)' \ + 'python3dist(terminaltables)' \ + 'python3dist(tensorflow-estimator)' \ + 'python3dist(tenacity)' \ + 'python3dist(smart-open)' \ + 'python3dist(ruamel.yaml.clib)' \ + 'python3dist(pyquaternion)' \ + 'python3dist(pyflakes)' \ + 'python3dist(pycodestyle)' \ + 'python3dist(prettytable)' \ + 'python3dist(nodeenv)' \ + 'python3dist(murmurhash)' \ + 'python3dist(mock)' \ + 'python3dist(mccabe)' \ + 'python3dist(langcodes)' \ + 'python3dist(itsdangerous)' \ + 'python3dist(isort)' \ + 'python3dist(iopath)' \ + 'python3dist(identify)' \ + 'python3dist(humanfriendly)' \ + 'python3dist(h5py)' \ + 'python3dist(h11)' \ + 'python3dist(greenlet)' \ + 'python3dist(fastprogress)' \ + 'python3dist(fastcore)' \ + 'python3dist(et-xmlfile)' \ + 'python3dist(dnspython)' \ + 'python3dist(cfgv)' \ + 'python3dist(blinker)' \ + 'python3dist(biopython)' \ + 'python3dist(beartype)' \ + 'python3dist(ruamel.yaml)' \ + 'python3dist(pre-commit)' \ + 'python3dist(plotly)' \ + 'python3dist(openpyxl)' \ + 'python3dist(fvcore)' \ + 'python3dist(flake8)' \ + 'python3dist(fastdownload)' \ + 'python3dist(eventlet)' \ + 'python3dist(coloredlogs)' \ + 'python3dist(botocore)' \ + 'python3dist(seqeval)' \ + 'python3dist(s3transfer)' \ + 'python3dist(qudida)' \ + 'python3dist(ptflops)' \ + 'python3dist(lyft-dataset-sdk)' \ + 'python3dist(hyperpyyaml)' \ + 'python3dist(descartes)' \ + 'python3dist(speechbrain)' \ + 'python3dist(pycocoevalcap)' \ + 'python3dist(ipywidgets)' \ + 'python3dist(boto3)' \ + 'python3dist(notebook)' \ + 'python3dist(pytorch3d)' && yum clean all + +RUN pip3 freeze > /var/modelscope/base.txt + +RUN sed -i '/datasets/d' /var/modelscope/framework.txt && \ + sed -i '/yapf/d' /var/modelscope/tests.txt && \ + sed -i '/timm/d' /var/modelscope/base.txt && \ + sed -i '/builddir/d' /var/modelscope/base.txt && \ + sed -i '/ffmpeg/d' /var/modelscope/cv.txt && \ + sed -i '/protobuf/d' /var/modelscope/nlp.txt && \ + sed -i '/decorator/d' /var/modelscope/*.txt && \ + sed -i '/scikit-image/d' /var/modelscope/*.txt + +RUN pip install --no-cache \ + -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html \ + -r /var/modelscope/framework.txt \ + -r /var/modelscope/audio.txt \ + -r /var/modelscope/cv.txt \ + -r /var/modelscope/multi-modal.txt \ + -r /var/modelscope/nlp.txt \ + -r /var/modelscope/science.txt \ + -r /var/modelscope/tests.txt \ + -r /var/modelscope/base.txt \ + spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy torchsde healpy + +RUN curl -L https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/modelscope/modelscope-1.12.0.tar.gz | tar -xz -C /root/ +RUN git clone https://www.modelscope.cn/ModelScope_Developer/modelscope-library-test-data.git /tmp/modelscope_test && \ + mv /tmp/modelscope_test/* /root/modelscope-1.12.0/data/test/ && \ + rm -rf /tmp/modelscope_test + +RUN cp -r /etc/yum.repos.d.bak/* /etc/yum.repos.d/ && cp /etc/pip.conf.bak /etc/pip.conf diff --git a/AI/modelscope/1.12.0/23/pip.conf b/AI/modelscope/1.12.0/23/pip.conf new file mode 100644 index 0000000000000000000000000000000000000000..ad0aefa1b90ef45edb6e6e0f62819b965299e3c0 --- /dev/null +++ b/AI/modelscope/1.12.0/23/pip.conf @@ -0,0 +1,3 @@ +[global] +index-url = http://mirrors.cloud.aliyuncs.com/pypi/web/simple +trusted-host = mirrors.cloud.aliyuncs.com \ No newline at end of file diff --git a/AI/modelscope/1.12.0/23/requirements/audio.txt b/AI/modelscope/1.12.0/23/requirements/audio.txt new file mode 100644 index 0000000000000000000000000000000000000000..88e469ceaa892f9e30329f20a1c3c40e6a099d5a --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio.txt @@ -0,0 +1,5 @@ +-r audio/audio_asr.txt +-r audio/audio_kws.txt +-r audio/audio_signal.txt +-r audio/audio_tts.txt +-r audio/audio_codec.txt diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_asr.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_asr.txt new file mode 100644 index 0000000000000000000000000000000000000000..a63614fe524e553d2a908dbc4f0cbf0e6a87002c --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_asr.txt @@ -0,0 +1 @@ +funasr>=1.0.0 diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_codec.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_codec.txt new file mode 100644 index 0000000000000000000000000000000000000000..c7ac8b2bd9fe206fdcb961f34b39b1948fa0307b --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_codec.txt @@ -0,0 +1 @@ +funcodec>=0.2.0 diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_kws.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_kws.txt new file mode 100644 index 0000000000000000000000000000000000000000..276a0a2f598f64272132d20f210d55c589b7b8af --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_kws.txt @@ -0,0 +1,7 @@ +kaldiio +kwsbp>=0.0.6 +matplotlib +py_sound_connect>=0.1 +scipy +SoundFile>0.10 +tensorboardX diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_signal.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_signal.txt new file mode 100644 index 0000000000000000000000000000000000000000..65f1ec61b6af856e09d8a54694e5b1ad37a90edc --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_signal.txt @@ -0,0 +1,12 @@ +hdbscan +hyperpyyaml +librosa==0.10.1 +MinDAEC +mir_eval>=0.7 +rotary_embedding_torch>=0.1.5 +scipy +SoundFile>0.10 +speechbrain>=0.5.12 +torchaudio +tqdm +umap-learn diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_tts.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_tts.txt new file mode 100644 index 0000000000000000000000000000000000000000..5cff1b2897731b9aed41d8f40d62ab37d8b67e91 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_tts.txt @@ -0,0 +1,28 @@ +bitstring +greenlet>=1.1.2 +inflect +jedi>=0.18.1 +kantts +librosa==0.10.1 +lxml +matplotlib +msgpack>=1.0.4 +parso>=0.8.3 +pexpect>=4.8.0 +pickleshare>=0.7.5 +prompt-toolkit>=3.0.30 +protobuf +ptflops +ptyprocess>=0.7.0 +pygments>=2.12.0 +pysptk>=0.1.15,<0.1.19 +pytorch_wavelets +PyWavelets>=1.0.0 +scikit-learn +sox +tensorboardx +tqdm +traitlets>=5.3.0 +ttsfrd>=0.1.2 +unidecode +wcwidth>=0.2.5 diff --git a/AI/modelscope/1.12.0/23/requirements/cv.txt b/AI/modelscope/1.12.0/23/requirements/cv.txt new file mode 100644 index 0000000000000000000000000000000000000000..c8edb672bf561c968c42bfc43fb31d765cd872ed --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/cv.txt @@ -0,0 +1,74 @@ +accelerate +albumentations>=1.0.3 +av>=9.2.0 +bmt_clipit>=1.0 +chumpy +clip>=1.0 +control_ldm +ddpm_guided_diffusion +diffusers +easydict +easyrobust +edit_distance +face_alignment>=1.3.5 +fairscale>=0.4.1 +fastai>=1.0.51 +ffmpeg>=1.4 +ffmpeg-python>=0.2.0 +ftfy +fvcore +# remove for windows support +# healpy +imageio>=2.9.0 +imageio-ffmpeg>=0.4.2 +imgaug>=0.4.0 +kornia>=0.5.0 +lap +lmdb +lpips +ml_collections +mmcls>=0.21.0 +mmdet>=2.25.0,<=2.28.2 +# mmdet3d-1.0.0rc6 remove networkx and numba version restriction +mmdet3d==1.0.0a1 +mmsegmentation<=0.30.0 +moviepy>=1.0.3 +nerfacc==0.2.2 +networkx +numba +omegaconf +onnx +onnxruntime>=1.10 +onnxsim +open-clip-torch>=2.7.0 +opencv-python +paint_ldm +pandas +panopticapi +plyfile>=0.7.4 +psutil +pyclipper +PyMCubes +pytorch-lightning +regex +# <0.20.0 for compatible python3.7 python3.8 +scikit-image>=0.19.3,<0.20.0 +scikit-learn>=0.20.1 +shapely +shotdetect_scenedetect_lgss>=0.0.4 +smplx +tensorflow-estimator>=1.15.1 +tf_slim +thop +timm>=0.4.9 +torch-scatter +torchmetrics>=0.6.2 +torchsummary>=1.5.1 +torchvision +tqdm +transformers>=4.26.0 +trimesh +ujson +utils +videofeatures_clipit>=1.0 +yacs diff --git a/AI/modelscope/1.12.0/23/requirements/docs.txt b/AI/modelscope/1.12.0/23/requirements/docs.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b3538351c05cbe6b973e26d3bf52bc14871a3a2 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/docs.txt @@ -0,0 +1,7 @@ +docutils>=0.16.0 +myst_parser +recommonmark +sphinx>=5.3.0 +sphinx-book-theme +sphinx-copybutton +sphinx_markdown_tables diff --git a/AI/modelscope/1.12.0/23/requirements/framework.txt b/AI/modelscope/1.12.0/23/requirements/framework.txt new file mode 100644 index 0000000000000000000000000000000000000000..8804fe8c7ffb57f8896f7963bbac85a76a5eec9b --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/framework.txt @@ -0,0 +1,22 @@ +addict +attrs +datasets>=2.14.5 +einops +filelock>=3.3.0 +gast>=0.2.2 +numpy +oss2 +pandas +Pillow>=6.2.0 +# pyarrow 9.0.0 introduced event_loop core dump +pyarrow>=6.0.0,!=9.0.0 +python-dateutil>=2.1 +pyyaml +requests>=2.25 +scipy +setuptools +simplejson>=3.3.0 +sortedcontainers>=1.5.9 +tqdm>=4.64.0 +urllib3>=1.26 +yapf diff --git a/AI/modelscope/1.12.0/23/requirements/multi-modal.txt b/AI/modelscope/1.12.0/23/requirements/multi-modal.txt new file mode 100644 index 0000000000000000000000000000000000000000..b4d551c2ddf96abba68a715f00916397c2564965 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/multi-modal.txt @@ -0,0 +1,29 @@ +accelerate +cloudpickle +decord>=0.6.0 +diffusers>=0.25.0 +fairseq +ftfy>=6.0.3 +librosa==0.10.1 +opencv-python +pycocoevalcap>=1.2 +pycocotools>=2.0.4 +pydot +# compatible with taming-transformers-rom1504 +pytorch_lightning<=1.7.7 +rapidfuzz +# rough-score was just recently updated from 0.0.4 to 0.0.7 +# which introduced compatability issues that are being investigated +rouge_score<=0.0.4 +sacrebleu +safetensors +# scikit-video +soundfile +taming-transformers-rom1504 +timm +tokenizers +torchvision +transformers>=4.27.1 +# triton==2.0.0.dev20221120 +unicodedata2 +zhconv diff --git a/AI/modelscope/1.12.0/23/requirements/nlp.txt b/AI/modelscope/1.12.0/23/requirements/nlp.txt new file mode 100644 index 0000000000000000000000000000000000000000..f69f869b83edf1c977f6a79d06657cbd23cfcbe8 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/nlp.txt @@ -0,0 +1,26 @@ +boto3 +embeddings +filelock +ftfy +jieba>=0.42.1 +matplotlib +megatron_util +nltk +pandas +# protobuf version beyond 3.20.0 is not compatible with TensorFlow 1.x, therefore is discouraged. +protobuf>=3.19.0,<3.21.0 +pythainlp +pyvi +regex +rouge +sacremoses>=0.0.41 +scikit_learn +sentencepiece +seqeval +spacy>=2.3.5 +stanza +subword_nmt>=0.3.8 +termcolor +tokenizers +transformers>=4.12.0 +zhconv diff --git a/AI/modelscope/1.12.0/23/requirements/science.txt b/AI/modelscope/1.12.0/23/requirements/science.txt new file mode 100644 index 0000000000000000000000000000000000000000..c30ff644539189e83e82dd322773cb2a21a14eed --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/science.txt @@ -0,0 +1,8 @@ +biopython +iopath +ipdb +lmdb +ml_collections +scipy +tensorboardX +tokenizers diff --git a/AI/modelscope/1.12.0/23/requirements/svr.txt b/AI/modelscope/1.12.0/23/requirements/svr.txt new file mode 100644 index 0000000000000000000000000000000000000000..ea439c6655ef1a23951dda7d12f1bfa4b52e7c2d --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/svr.txt @@ -0,0 +1,4 @@ +fastapi +requests +sse-starlette +uvicorn diff --git a/AI/modelscope/1.12.0/23/requirements/tensorflow1x.txt b/AI/modelscope/1.12.0/23/requirements/tensorflow1x.txt new file mode 100644 index 0000000000000000000000000000000000000000..c808f28fc4e20f1082633eefcbd65854cecec5c8 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/tensorflow1x.txt @@ -0,0 +1 @@ +numpy<=1.18.5 diff --git a/AI/modelscope/1.12.0/23/requirements/tests.txt b/AI/modelscope/1.12.0/23/requirements/tests.txt new file mode 100644 index 0000000000000000000000000000000000000000..5ec4df7e8f107223a5f25e90eddefddf0e15f7db --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/tests.txt @@ -0,0 +1,5 @@ +expecttest +flake8 +isort>=4.3.21 +pre-commit +yapf==0.30.0 # use fix version to ensure consistent auto-styling diff --git a/AI/modelscope/buildspec.yml b/AI/modelscope/buildspec.yml index 8604d20318be8b8fba61b14b46c729d1355642a6..4d6a87b9019ebc7aa8a248787975dd01c793e156 100644 --- a/AI/modelscope/buildspec.yml +++ b/AI/modelscope/buildspec.yml @@ -16,7 +16,7 @@ t-one: test_suite: &TEST_SUITE image-ci-test # 执行测试 case, 多个用数组表示 test_conf: &TEST_CONF group=ai_container - test_case: &TEST_CASE pytorch_ai_container,tensorflow2_ai_container,jupyterlab_ai_container + test_case: &TEST_CASE pytorch_ai_container cloud_server_tag: &CLOUD_SERVER_TAG [anolis-container-ci-x86, anolis-container-ci-arm] # 构建镜像配置 @@ -26,14 +26,29 @@ images: # 定义是否构建参数 build: true test: false - platform: [linux/amd64] + platform: [linux/amd64] docker_file: path: AI/modelscope/1.10.0/8/Dockerfile - variable: + variable: - &GPU [(USE_GPU, "True")] scene: args: [*GPU] - tags: + tags: - [1.10.0-an8] registry: [*ACR_PROD] test_config: [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] + + BuildA23ModelscopeDockerImage: + # 定义是否构建参数 + build: true + platform: [linux/amd64] + docker_file: + path: AI/modelscope/1.12.0/23/Dockerfile + variable: + - &GPU [(USE_GPU, "True")] + scene: + args: [*GPU] + tags: + - [1.12.0-an23-cu121-torch201] + registry: [*ACR_PROD] + test_config: [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] \ No newline at end of file