Skip to content

Commit 50d74b5

Browse files
SunsetWolfSunsetWolf
authored
split_CI (#1141)
1 parent a87b026 commit 50d74b5

File tree

12 files changed

+174
-143
lines changed

12 files changed

+174
-143
lines changed

‎.github/workflows/test_macos.yml‎

Lines changed: 0 additions & 94 deletions
This file was deleted.

‎.github/workflows/test_qlib_from_pip.yml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Test qlib from pip
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
timeout-minutes: 120
12+
13+
runs-on: ${{ matrix.os }}
14+
strategy:
15+
matrix:
16+
os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
17+
# not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
18+
python-version: [3.7, 3.8]
19+
20+
steps:
21+
- name: Test qlib from pip
22+
uses: actions/checkout@v2
23+
24+
- name: Set up Python ${{ matrix.python-version }}
25+
uses: actions/setup-python@v2
26+
with:
27+
python-version: ${{ matrix.python-version }}
28+
29+
- name: Update pip to the latest version
30+
run: |
31+
python -m pip install --upgrade pip
32+
33+
- name: Qlib installation test
34+
run: |
35+
python -m pip install pyqlib
36+
# Specify the numpy version because the numpy upgrade caused the CI test to fail,
37+
# and this line of code will be removed when the next version of qlib is released.
38+
python -m pip install "numpy<1.23"
39+
40+
- name: Downloads dependencies data
41+
run: |
42+
python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
43+
44+
- name: Test workflow by config
45+
run: |
46+
qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

‎.github/workflows/test.yml‎ renamed to ‎.github/workflows/test_qlib_from_source.yml‎

Lines changed: 42 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Test
1+
name: Test qlib from source
22

33
on:
44
push:
@@ -9,41 +9,60 @@ on:
99
jobs:
1010
build:
1111
timeout-minutes: 120
12+
1213
runs-on: ${{ matrix.os }}
1314
strategy:
1415
matrix:
15-
os: [windows-latest, ubuntu-18.04, ubuntu-20.04]
16+
os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
1617
# not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
1718
python-version: [3.7, 3.8]
1819

1920
steps:
20-
- uses: actions/checkout@v2
21+
- name: Test qlib from source
22+
uses: actions/checkout@v2
2123

2224
- name: Set up Python ${{ matrix.python-version }}
2325
uses: actions/setup-python@v2
2426
with:
2527
python-version: ${{ matrix.python-version }}
2628

27-
- name: Lint with Black
29+
- name: Update pip to the latest version
30+
run: |
31+
python -m pip install --upgrade pip
32+
33+
- name: Installing pytorch for macos
34+
if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
35+
run: |
36+
python -m pip install torch torchvision torchaudio
37+
brew install libomp # lightgbm dependencies
38+
39+
- name: Installing pytorch for ubuntu
40+
if: ${{ matrix.os == 'ubuntu-18.04' || matrix.os == 'ubuntu-20.04' }}
2841
run: |
29-
pip install --upgrade pip
30-
pip install black wheel
31-
black qlib -l 120 --check --diff
42+
python -m pip install --upgrade pip
43+
python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
3244
33-
- name: Install Qlib with pip
45+
- name: Installing pytorch for windows
46+
if: ${{ matrix.os == 'windows-latest' }}
47+
run: |
48+
python -m pip install --upgrade pip
49+
python -m pip install torch torchvision torchaudio
50+
51+
- name: Set up Python tools
52+
run: |
53+
python -m pip install --upgrade cython
54+
python -m pip install -e .[dev]
55+
56+
- name: Lint with Black
3457
run: |
35-
pip install numpy==1.19.5 ruamel.yaml
36-
pip install pyqlib --ignore-installed
58+
black . -l 120 --check --diff
3759
3860
- name: Make html with sphinx
3961
run: |
40-
pip install -U sphinx
41-
pip install sphinx_rtd_theme readthedocs_sphinx_ext
42-
pip install --exists-action=w --no-cache-dir -r docs/requirements.txt
4362
cd docs
4463
sphinx-build -b html . build
4564
cd ..
46-
65+
4766
# Check Qlib with pylint
4867
# TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
4968
# C0103: invalid-name
@@ -67,11 +86,9 @@ jobs:
6786
# W1309: f-string-without-interpolation
6887
# E1102: not-callable
6988
# E1136: unsubscriptable-object
70-
# References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962
89+
# References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962
7190
- name: Check Qlib with pylint
7291
run: |
73-
pip install --upgrade pip
74-
pip install pylint
7592
pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500"
7693
7794
# The following flake8 error codes were ignored:
@@ -95,47 +112,29 @@ jobs:
95112
# Description: If there is whitespace before ":", it cannot pass the black check.
96113
- name: Check Qlib with flake8
97114
run: |
98-
pip install --upgrade pip
99-
pip install flake8
100115
flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib
101116
102117
# https://github.com/python/mypy/issues/10600
103118
- name: Check Qlib with mypy
104119
run: |
105-
pip install mypy
106120
mypy qlib --install-types --non-interactive || true
107-
mypy qlib
121+
mypy qlib --verbose
108122
109123
- name: Test data downloads
110124
run: |
111-
python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data_simple --interval 1d --region cn
112-
python -c "import os; userpath=os.path.expanduser('~'); os.rename(userpath + '/.qlib/qlib_data/cn_data_simple', userpath + '/.qlib/qlib_data/cn_data')"
125+
python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
113126
azcopy copy https://qlibpublic.blob.core.windows.net/data/rl /tmp/qlibpublic/data --recursive
114127
mv /tmp/qlibpublic/data tests/.data
115128
116-
- name: Test workflow by config (install from pip)
129+
- name: Test workflow by config (install from source)
117130
run: |
131+
# Version 0.52.0 of numba must be installed manually in CI, otherwise it will cause incompatibility with the latest version of numpy.
132+
python -m pip install numba==0.52.0
133+
# You must update numpy manually, because when installing python tools, it will try to uninstall numpy and cause CI to fail.
134+
python -m pip install --upgrade numpy
118135
python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
119-
python -m pip uninstall -y pyqlib
120-
121-
# Test Qlib installed from source
122-
- name: Install Qlib from source
123-
run: |
124-
pip install --upgrade cython jupyter jupyter_contrib_nbextensions numpy scipy scikit-learn # installing without this line will cause errors on GitHub Actions, while instsalling locally won't
125-
pip install gym tianshou torch
126-
pip install -e .
127-
128-
- name: Install test dependencies
129-
run: |
130-
pip install --upgrade pip
131-
pip install black pytest
132136
133137
- name: Unit tests with Pytest
134138
run: |
135-
pip install -r scripts/data_collector/pit/requirements.txt
136139
cd tests
137-
python -m pytest . --durations=10
138-
139-
- name: Test workflow by config (install from source)
140-
run: |
141-
python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml
140+
python -m pytest . -m "not slow" --durations=0

‎.github/workflows/test_qlib_from_source_slow.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: Test qlib from source slow
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
timeout-minutes: 120
12+
13+
runs-on: ${{ matrix.os }}
14+
strategy:
15+
matrix:
16+
os: [windows-latest, ubuntu-18.04, ubuntu-20.04, macos-11, macos-latest]
17+
# not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
18+
python-version: [3.7, 3.8]
19+
20+
steps:
21+
- name: Test qlib from source slow
22+
uses: actions/checkout@v2
23+
24+
- name: Set up Python ${{ matrix.python-version }}
25+
uses: actions/setup-python@v2
26+
with:
27+
python-version: ${{ matrix.python-version }}
28+
29+
- name: Set up Python tools
30+
run: |
31+
pip install --upgrade cython numpy pip
32+
pip install -e .[dev]
33+
34+
- name: Downloads dependencies data
35+
run: |
36+
python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
37+
38+
- name: Install Lightgbm for MacOS
39+
if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
40+
run: |
41+
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
42+
HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
43+
# FIX MacOS error: Segmentation fault
44+
# reference: https://github.com/microsoft/LightGBM/issues/4229
45+
wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
46+
brew unlink libomp
47+
brew install libomp.rb
48+
49+
- name: Unit tests with Pytest
50+
run: |
51+
cd tests
52+
python -m pytest . -m "slow" --durations=0

‎qlib/data/storage/file_storage.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,10 +106,7 @@ def _read_calendar(self, skip_rows: int = 0, n_rows: int = None) -> List[CalVT]:
106106
if not self.uri.exists():
107107
self._write_calendar(values=[])
108108
with self.uri.open("rb") as fp:
109-
return [
110-
str(x)
111-
for x in np.loadtxt(fp, str, skiprows=skip_rows, max_rows=n_rows, delimiter="\n", encoding="utf-8")
112-
]
109+
return [str(x) for x in np.loadtxt(fp, str, skiprows=skip_rows, max_rows=n_rows, encoding="utf-8")]
113110

114111
def _write_calendar(self, values: Iterable[CalVT], mode: str = "wb"):
115112
with self.uri.open(mode=mode) as fp:

‎setup.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,10 +137,24 @@ def get_version(rel_path: str) -> str:
137137
"sphinx",
138138
"sphinx_rtd_theme",
139139
"pre-commit",
140+
# CI dependencies
141+
"wheel",
142+
"setuptools",
143+
"black",
144+
"pylint",
145+
"mypy",
146+
"flake8",
147+
"readthedocs_sphinx_ext",
148+
"cmake",
149+
"lxml",
150+
"baostock",
151+
"yahooquery",
152+
"beautifulsoup4",
153+
"tianshou",
154+
"gym>=0.24", # If you do not put gym at the end, gym will degrade causing pytest results to fail.
140155
],
141156
"rl": [
142157
"tianshou",
143-
"gym",
144158
"torch",
145159
],
146160
},

‎tests/pytest.ini‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
[pytest]
2+
markers =
3+
slow: marks tests as slow (deselect with '-m "not slow"')
24
filterwarnings =
35
ignore:.*rng.randint:DeprecationWarning
46
ignore:.*Casting input x to numpy array:UserWarning

0 commit comments

Comments
 (0)