Skip to content

Commit f7b1106

Browse files
xuhancnxuhancn
authored
Merge branch 'main' into xu_fix_BUILD_SEPARATE_OPS
2 parents ffdf35b + 8fa9749 commit f7b1106

File tree

267 files changed

+13005
-5032
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

267 files changed

+13005
-5032
lines changed

‎.ci/docker/README.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Docker images for GitHub CI and CD
2+
3+
This directory contains everything needed to build the Docker images
4+
that are used in our CI tests.
5+
6+
## Docker CI builds
7+
8+
* `pytorch/manylinux2_28-builder:xpu-main` -- can use pytorch CICD image directly
9+
10+
## Docker CI tests
11+
12+
If also use this for build, need install Intel® Deep Learning Essentials,
13+
refer to [Intel® Deep Learning Essentials](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit-download.html?packages=dl-essentials&dl-essentials-os=linux&dl-lin=offline)
14+
```bash
15+
# Build a specific image for tests
16+
docker build --build-arg UBUNTU_VERSION=22.04 --file ubuntu/Dockerfile --build-arg XPU_DRIVER_TYPE=LTS2 .
17+
```

‎.ci/docker/common/install_xpu.sh‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
#!/bin/bash
2+
set -xe
3+
# Script used in CI and CD pipeline
4+
5+
# Intel® software for general purpose GPU capabilities.
6+
# Refer to https://www.intel.com/content/www/us/en/developer/articles/tool/pytorch-prerequisites-for-intel-gpus.html
7+
8+
# Users should update to the latest version as it becomes available
9+
10+
function install_ubuntu() {
11+
. /etc/os-release
12+
if [ "${XPU_DRIVER_TYPE,,}" == "lts" ]; then
13+
if [[ ! " jammy " =~ " ${VERSION_CODENAME} " ]]; then
14+
echo "Ubuntu version ${VERSION_CODENAME} with ${XPU_DRIVER_TYPE} not supported"
15+
exit 1
16+
fi
17+
else
18+
if [[ ! " jammy noble " =~ " ${VERSION_CODENAME} " ]]; then
19+
echo "Ubuntu version ${VERSION_CODENAME} with ${XPU_DRIVER_TYPE} not supported"
20+
exit 1
21+
fi
22+
fi
23+
24+
apt-get update -y
25+
apt-get install -y gpg-agent wget
26+
# To add the online network package repository for the GPU Driver
27+
wget -qO - https://repositories.intel.com/gpu/intel-graphics.key \
28+
| gpg --yes --dearmor --output /usr/share/keyrings/intel-graphics.gpg
29+
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] \
30+
https://repositories.intel.com/gpu/ubuntu ${VERSION_CODENAME}${XPU_DRIVER_VERSION} unified" \
31+
| tee /etc/apt/sources.list.d/intel-gpu-${VERSION_CODENAME}.list
32+
33+
# Update the packages list and repository index
34+
apt-get update
35+
36+
# The xpu-smi packages
37+
apt-get install -y flex bison xpu-smi
38+
if [ "${XPU_DRIVER_TYPE,,}" == "lts" ]; then
39+
# Compute and Media Runtimes
40+
apt-get install -y \
41+
intel-opencl-icd intel-level-zero-gpu level-zero \
42+
intel-media-va-driver-non-free libmfx1 libmfxgen1 libvpl2 \
43+
libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
44+
libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
45+
mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo
46+
# Development Packages
47+
apt-get install -y libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev level-zero-dev
48+
else # rolling or lts2 driver
49+
if [ "${VERSION_CODENAME}" == "jammy" ];then
50+
apt-get install -y \
51+
intel-opencl-icd libze-intel-gpu1 libze1 \
52+
intel-media-va-driver-non-free libmfx-gen1 libvpl2 \
53+
libegl-mesa0 libegl1-mesa libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
54+
libglapi-mesa libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
55+
mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo intel-ocloc
56+
else
57+
apt-get install -y \
58+
intel-opencl-icd libze-intel-gpu1 libze1 \
59+
intel-media-va-driver-non-free libmfx-gen1 libvpl2 \
60+
libegl-mesa0 libegl1-mesa-dev libgbm1 libgl1-mesa-dev libgl1-mesa-dri \
61+
libglapi-mesa libgles2-mesa-dev libglx-mesa0 libigdgmm12 libxatracker2 mesa-va-drivers \
62+
mesa-vdpau-drivers mesa-vulkan-drivers va-driver-all vainfo hwinfo clinfo intel-ocloc
63+
fi
64+
apt-get install -y libigc-dev intel-igc-cm libigdfcl-dev libigfxcmrt-dev libze-dev
65+
fi
66+
67+
# Cleanup
68+
apt-get autoclean && apt-get clean
69+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
70+
}
71+
72+
function install_rhel() {
73+
. /etc/os-release
74+
if [ "${XPU_DRIVER_TYPE,,}" == "lts" ]; then
75+
if [[ "${ID}" == "rhel" ]]; then
76+
if [[ ! " 8.8 8.10 9.2 9.4 9.5 " =~ " ${VERSION_ID} " ]]; then
77+
echo "RHEL version ${VERSION_ID} with ${XPU_DRIVER_TYPE} not supported"
78+
exit 1
79+
fi
80+
elif [[ "${ID}" == "almalinux" ]]; then
81+
# Workaround for almalinux8 which used by quay.io/pypa/manylinux_2_28_x86_64
82+
VERSION_ID="8.8"
83+
fi
84+
elif [ "${XPU_DRIVER_TYPE,,}" == "lts2" ]; then
85+
if [[ "${ID}" == "rhel" ]]; then
86+
if [[ ! " 8.10 9.4 9.6 10.0 " =~ " ${VERSION_ID} " ]]; then
87+
echo "RHEL version ${VERSION_ID} with ${XPU_DRIVER_TYPE} not supported"
88+
exit 1
89+
fi
90+
elif [[ "${ID}" == "almalinux" ]]; then
91+
# Workaround for almalinux8 which used by quay.io/pypa/manylinux_2_28_x86_64
92+
VERSION_ID="8.10"
93+
fi
94+
else # rolling driver
95+
if [[ "${ID}" == "rhel" ]]; then
96+
if [[ ! " 8.10 9.4 9.6 " =~ " ${VERSION_ID} " ]]; then
97+
echo "RHEL version ${VERSION_ID} with ${XPU_DRIVER_TYPE} not supported"
98+
exit 1
99+
fi
100+
elif [[ "${ID}" == "almalinux" ]]; then
101+
# Workaround for almalinux8 which used by quay.io/pypa/manylinux_2_28_x86_64
102+
VERSION_ID="8.10"
103+
fi
104+
fi
105+
106+
dnf install -y 'dnf-command(config-manager)'
107+
# To add the online network package repository for the GPU Driver
108+
dnf config-manager --add-repo \
109+
https://repositories.intel.com/gpu/rhel/${VERSION_ID}${XPU_DRIVER_VERSION}/unified/intel-gpu-${VERSION_ID}.repo
110+
111+
# Install Intel Support Packages
112+
yum install -y ${XPU_PACKAGES}
113+
# The xpu-smi packages
114+
dnf install -y flex bison xpu-smi
115+
# Compute and Media Runtimes
116+
if [ "${XPU_DRIVER_TYPE,,}" == "lts" ]; then
117+
dnf install --skip-broken -y \
118+
intel-opencl intel-media libmfxgen1 libvpl2 \
119+
level-zero intel-level-zero-gpu mesa-dri-drivers mesa-vulkan-drivers \
120+
mesa-vdpau-drivers mesa-libEGL mesa-libgbm mesa-libGL \
121+
mesa-libxatracker libvpl-tools intel-metrics-discovery \
122+
intel-metrics-library intel-igc-core intel-igc-cm \
123+
libva libva-utils intel-gmmlib libmetee intel-gsc intel-ocloc
124+
else
125+
dnf install --skip-broken -y \
126+
intel-opencl intel-media libmfxgen1 libvpl2 \
127+
level-zero intel-level-zero-gpu mesa-dri-drivers mesa-vulkan-drivers \
128+
mesa-vdpau-drivers libdrm mesa-libEGL mesa-libgbm mesa-libGL \
129+
mesa-libxatracker libvpl-tools intel-metrics-discovery \
130+
intel-metrics-library intel-igc-core intel-igc-cm \
131+
libva libva-utils intel-gmmlib libmetee intel-gsc intel-ocloc
132+
fi
133+
# Development packages
134+
dnf install -y --refresh intel-igc-opencl-devel level-zero-devel intel-gsc-devel libmetee-devel
135+
dnf install --enablerepo epel -y hwinfo clinfo
136+
137+
# Cleanup
138+
dnf clean all
139+
rm -rf /var/cache/yum
140+
rm -rf /var/lib/yum/yumdb
141+
rm -rf /var/lib/yum/history
142+
}
143+
144+
function install_sles() {
145+
. /etc/os-release
146+
VERSION_SP=${VERSION_ID//./sp}
147+
if [ "${XPU_DRIVER_TYPE,,}" == "lts" ]; then
148+
if [[ ! " 15sp4 15sp5 15sp6 " =~ " ${VERSION_SP} " ]]; then
149+
echo "SLES version ${VERSION_ID} with ${XPU_DRIVER_TYPE} not supported"
150+
exit
151+
fi
152+
elif [ "${XPU_DRIVER_TYPE,,}" == "lts2" ]; then
153+
if [[ ! " 15sp4 15sp5 15sp6 15sp7" =~ " ${VERSION_SP} " ]]; then
154+
echo "SLES version ${VERSION_ID} with ${XPU_DRIVER_TYPE} not supported"
155+
exit
156+
fi
157+
else # rolling
158+
if [[ ! " 15sp4 15sp5 15sp6 " =~ " ${VERSION_SP} " ]]; then
159+
echo "SLES version ${VERSION_ID} with ${XPU_DRIVER_TYPE} not supported"
160+
exit
161+
fi
162+
fi
163+
164+
# To add the online network package repository for the GPU Driver
165+
zypper addrepo -f -r \
166+
https://repositories.intel.com/gpu/sles/${VERSION_SP}${XPU_DRIVER_VERSION}/unified/intel-gpu-${VERSION_SP}.repo
167+
rpm --import https://repositories.intel.com/gpu/intel-graphics.key
168+
169+
# The xpu-smi packages
170+
zypper install -y lsb-release flex bison xpu-smi
171+
# Compute and Media Runtimes
172+
zypper install -y \
173+
intel-level-zero-gpu level-zero intel-gsc intel-opencl intel-ocloc \
174+
intel-media-driver libigfxcmrt7 libvpl2 libvpl-tools libmfxgen1
175+
if [ "${XPU_DRIVER_TYPE,,}" == "lts" ]; then
176+
zypper install -y libmfx1
177+
fi
178+
# Development packages
179+
zypper install -y libigdfcl-devel intel-igc-cm libigfxcmrt-devel level-zero-devel
180+
zypper install -y clinfo libOpenCL1 libva-utils hwinfo
181+
182+
}
183+
184+
# Default use GPU driver rolling releases
185+
XPU_DRIVER_VERSION=""
186+
if [ "${XPU_DRIVER_TYPE,,}" == "lts" ]; then
187+
# Use GPU driver LTS releases
188+
XPU_DRIVER_VERSION="/lts/2350"
189+
elif [ "${XPU_DRIVER_TYPE,,}" == "lts2" ]; then
190+
# Use GPU driver LTS releases
191+
XPU_DRIVER_VERSION="/lts/2523"
192+
fi
193+
194+
# The installation depends on the base OS
195+
ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
196+
case "$ID" in
197+
ubuntu)
198+
install_ubuntu
199+
;;
200+
rhel|almalinux)
201+
install_rhel
202+
;;
203+
sles)
204+
install_sles
205+
;;
206+
*)
207+
echo "Unable to determine OS..."
208+
exit 1
209+
;;
210+
esac

‎.ci/docker/ubuntu/Dockerfile‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
ARG UBUNTU_VERSION
2+
3+
FROM ubuntu:${UBUNTU_VERSION}
4+
5+
ENV DEBIAN_FRONTEND noninteractive
6+
7+
# install Intel GPU driver LTS2, refer to https://dgpu-docs.intel.com/driver/installation-lts2.html
8+
ARG XPU_DRIVER_TYPE
9+
ENV XPU_DRIVER_TYPE ${XPU_DRIVER_TYPE}
10+
11+
COPY ./common/install_xpu.sh install_xpu.sh
12+
RUN bash ./install_xpu.sh && rm -f install_xpu.sh
13+
14+
# install extra packages for pytorch benchmark
15+
RUN apt-get update && \
16+
apt-get install -y wget curl sudo git unzip zip gh numactl rsync jq && \
17+
apt-get install -y gcc g++ cmake libgl1 zlib1g-dev libglib2.0-dev && \
18+
apt-get install -y libnl-genl-3-200 && \
19+
apt-get autoclean && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
20+
21+
22+
USER jenkins
23+
CMD ["bash"]

‎.github/ISSUE_TEMPLATE/dynamic-skip.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: 🐛 Dynamic skip
2+
description: Create an issue to skip PR unrelated failures dynamically
3+
title: "[Bug Skip]: "
4+
labels: ["skipped"]
5+
6+
body:
7+
- type: markdown
8+
attributes:
9+
value: >
10+
#### Before submitting a bug, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/pytorch/pytorch/issues?q=is%3Aissue+sort%3Acreated-desc+).
11+
- type: textarea
12+
attributes:
13+
label: 🐛 Describe the bug with skip template
14+
description: |
15+
Please provide a clear and concise description of what the bug is.
16+
The template for dynamic skip as below:
17+
18+
```python
19+
# Template(Check in the github action summary)
20+
Cases:
21+
[Category],[Class name],[Test name]
22+
```
23+
24+
```python
25+
# example
26+
Cases:
27+
op_ut,third_party.torch-xpu-ops.test.xpu.test_transformers_xpu.TestTransformersXPU,test_scaled
28+
```
29+
30+
If the code is too long (hopefully, it isn't), feel free to put it in a public gist and link it in the issue: https://gist.github.com.
31+
32+
Please also paste or describe the results you observe instead of the expected results. If you observe an error, please paste the error message including the **full** traceback of the exception. It may be relevant to wrap error messages in ```` ```triple quotes blocks``` ````.
33+
placeholder: |
34+
A clear and concise description of what the bug is and also align the dynamic template.
35+
```
36+
# Skippped cases with dynamic template
37+
```
38+
39+
```python
40+
# Sample code to reproduce the problem
41+
```
42+
43+
```
44+
The error message you got, with the full traceback.
45+
```
46+
validations:
47+
required: true
48+
- type: textarea
49+
attributes:
50+
label: Versions
51+
description: |
52+
Please run the following and paste the output below.
53+
```sh
54+
wget https://raw.githubusercontent.com/pytorch/pytorch/main/torch/utils/collect_env.py
55+
# For security purposes, please check the contents of collect_env.py before running it.
56+
python collect_env.py
57+
```
58+
validations:
59+
required: true
60+
- type: markdown
61+
attributes:
62+
value: >
63+
Thanks for contributing 🎉!

0 commit comments

Comments
 (0)