Skip to content

Commit 166e06c

Browse files
satrasatra
authored
Merge pull request #2386 from effigies/ci/workflow
TST/CI: Split Circle workflow
2 parents e0d829e + d424f91 commit 166e06c

14 files changed

+293
-144
lines changed

‎.circleci/config.yml

+259-89
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,121 @@
1+
_machine_kwds: &machine_kwds
2+
image: circleci/classic:201710-02
3+
4+
_store_artifacts_kwds: &store_artifacts_kwds
5+
path: /home/circleci/work/tests
6+
7+
_test_environment: &test_environment
8+
WORKDIR: /home/circleci/work
9+
DOCKER_IMAGE: "nipype/nipype"
10+
11+
_set_pr_number: &set_pr_number
12+
name: Set PR number
13+
command: |
14+
echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
15+
source $BASH_ENV
16+
echo $CIRCLE_PR_NUMBER
17+
18+
_generate_dockerfiles: &generate_dockerfiles
19+
name: Generate Dockerfiles
20+
command: |
21+
make gen-dockerfiles
22+
23+
_modify_nipype_version: &modify_nipype_version
24+
name: Modify Nipype version if necessary
25+
command: |
26+
if [ "$CIRCLE_TAG" != "" ]; then
27+
sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
28+
fi
29+
30+
_get_base_image: &get_base_image
31+
name: Get base image (pull or build)
32+
no_output_timeout: 60m
33+
command: |
34+
source /tmp/docker/get_base_image.sh
35+
if [ "$GET_BASE" == "PULL" ]; then
36+
echo "Pulling base image ..."
37+
docker pull nipype/nipype:base
38+
elif [ "$GET_BASE" == "BUILD" ]; then
39+
e=1 && for i in {1..5}; do
40+
docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15
41+
done && [ "$e" -eq "0" ]
42+
else
43+
echo "Error: method to get base image not understood"
44+
exit 1
45+
fi
46+
47+
_build_main_image_py36: &build_main_image_py36
48+
name: Build main image (py36)
49+
no_output_timeout: 60m
50+
command: |
51+
e=1 && for i in {1..5}; do
52+
docker build \
53+
--rm=false \
54+
--tag nipype/nipype:latest \
55+
--tag nipype/nipype:py36 \
56+
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
57+
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
58+
--build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
59+
&& e=0 && break || sleep 15
60+
done && [ "$e" -eq "0" ]
61+
62+
_build_main_image_py27: &build_main_image_py27
63+
name: Build main image (py27)
64+
no_output_timeout: 60m
65+
command: |
66+
e=1 && for i in {1..5}; do
67+
docker build \
68+
--rm=false \
69+
--tag nipype/nipype:py27 \
70+
--build-arg PYTHON_VERSION_MAJOR=2 \
71+
--build-arg PYTHON_VERSION_MINOR=7 \
72+
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
73+
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
74+
--build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
75+
&& e=0 && break || sleep 15
76+
done && [ "$e" -eq "0" ]
77+
78+
_download_test_data: &_download_test_data
79+
name: Download test data
80+
no_output_timeout: 20m
81+
working_directory: /home/circleci/examples
82+
environment:
83+
OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
84+
command: |
85+
export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
86+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj
87+
88+
export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
89+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz
90+
91+
export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
92+
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
93+
94+
_prepare_working_directory: &prepare_working_directory
95+
name: Prepare working directory
96+
environment: *test_environment
97+
command: |
98+
mkdir -p "$WORKDIR"
99+
chmod -R 777 "$WORKDIR"
100+
101+
_get_codecov: &_get_codecov
102+
name: Get codecov
103+
command: |
104+
pip install --no-cache-dir codecov
105+
106+
_run_codecov_coverage: &_run_codecov_coverage
107+
name: Run codecov (coverage)
108+
environment: *test_environment
109+
command: |
110+
codecov --file $WORKDIR/tests/coverage*.xml --root "$HOME/nipype" --flags unittests -e CIRCLE_JOB
111+
112+
_run_codecov_smoke: &_run_codecov_smoke
113+
name: Run codecov (smoke tests)
114+
environment: *test_environment
115+
command: |
116+
codecov --file $WORKDIR/tests/smoketest*.xml --root "$HOME/nipype" --flags smoketests -e CIRCLE_JOB
117+
118+
1119
version: 2
2120
jobs:
3121

@@ -35,116 +153,156 @@ jobs:
35153
- docker/Dockerfile.base-pruned
36154
- docker/get_base_image.sh
37155

156+
test_pytest:
157+
machine: *machine_kwds
158+
working_directory: /home/circleci/nipype
159+
steps:
160+
- checkout:
161+
path: /home/circleci/nipype
162+
- attach_workspace:
163+
at: /tmp
164+
- run: *set_pr_number
165+
- run: *generate_dockerfiles
166+
- run: *modify_nipype_version
167+
- run: *get_base_image
168+
- run: *build_main_image_py36
169+
- run: *build_main_image_py27
170+
- run: *_get_codecov
171+
- run: *_download_test_data
172+
- run: *prepare_working_directory
173+
- run:
174+
name: Run pytests (py36)
175+
no_output_timeout: 30m
176+
environment: *test_environment
177+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_pytest.sh
178+
- run:
179+
name: Run pytests (py27)
180+
no_output_timeout: 30m
181+
environment: *test_environment
182+
command: bash -ux /home/circleci/nipype/.circleci/test_py2_pytest.sh
183+
- run: *_run_codecov_coverage
184+
- store_artifacts: *store_artifacts_kwds
185+
- store_test_results: *store_artifacts_kwds
38186

39-
build_and_test:
40-
parallelism: 4
41-
machine:
42-
# Ubuntu 14.04 with Docker 17.10.0-ce
43-
image: circleci/classic:201710-02
187+
test_py3_fmri_fsl_spm:
188+
machine: *machine_kwds
44189
working_directory: /home/circleci/nipype
45190
steps:
46191
- checkout:
47192
path: /home/circleci/nipype
48193
- attach_workspace:
49194
at: /tmp
195+
- run: *set_pr_number
196+
- run: *generate_dockerfiles
197+
- run: *modify_nipype_version
198+
- run: *get_base_image
199+
- run: *build_main_image_py36
200+
- run: *_get_codecov
201+
- run: *_download_test_data
202+
- run: *prepare_working_directory
50203
- run:
51-
name: Get test dependencies and generate Dockerfiles
52-
command: |
53-
pip install --no-cache-dir codecov
54-
make gen-dockerfiles
204+
name: Run FSL FEEDS pipeline (py36)
205+
no_output_timeout: 40m
206+
environment: *test_environment
207+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_fsl_feeds_linear_l1.sh
55208
- run:
56-
name: Modify Nipype version if necessary
57-
command: |
58-
if [ "$CIRCLE_TAG" != "" ]; then
59-
sed -i -E "s/(__version__ = )'[A-Za-z0-9.-]+'/\1'$CIRCLE_TAG'/" nipype/info.py
60-
fi
209+
name: Run FSL reuse pipeline (py36)
210+
no_output_timeout: 40m
211+
environment: *test_environment
212+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_fsl_reuse_linear_l1.sh
61213
- run:
62-
name: Get base image (pull or build)
63-
no_output_timeout: 60m
64-
command: |
65-
source /tmp/docker/get_base_image.sh
66-
if [ "$GET_BASE" == "PULL" ]; then
67-
echo "Pulling base image ..."
68-
docker pull nipype/nipype:base
69-
elif [ "$GET_BASE" == "BUILD" ]; then
70-
e=1 && for i in {1..5}; do
71-
docker build -t nipype/nipype:base - < docker/Dockerfile.base && e=0 && break || sleep 15
72-
done && [ "$e" -eq "0" ]
73-
else
74-
echo "Error: method to get base image not understood"
75-
exit 1
76-
fi
214+
name: Run SPM test workflow - 3D inputs (py36)
215+
no_output_timeout: 40m
216+
environment: *test_environment
217+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_3d.sh
77218
- run:
78-
name: Build main image (py36)
79-
no_output_timeout: 60m
80-
command: |
81-
e=1 && for i in {1..5}; do
82-
docker build \
83-
--rm=false \
84-
--tag nipype/nipype:latest \
85-
--tag nipype/nipype:py36 \
86-
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
87-
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
88-
--build-arg VERSION="${CIRCLE_TAG}" /home/circleci/nipype \
89-
&& e=0 && break || sleep 15
90-
done && [ "$e" -eq "0" ]
91-
- run:
92-
name: Build main image (py27)
93-
no_output_timeout: 60m
94-
command: |
95-
e=1 && for i in {1..5}; do
96-
docker build \
97-
--rm=false \
98-
--tag nipype/nipype:py27 \
99-
--build-arg PYTHON_VERSION_MAJOR=2 \
100-
--build-arg PYTHON_VERSION_MINOR=7 \
101-
--build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
102-
--build-arg VCS_REF="$(git rev-parse --short HEAD)" \
103-
--build-arg VERSION="${CIRCLE_TAG}-py27" /home/circleci/nipype \
104-
&& e=0 && break || sleep 15
105-
done && [ "$e" -eq "0" ]
106-
- run:
107-
name: Download test data
108-
no_output_timeout: 20m
109-
working_directory: /home/circleci/examples
110-
environment:
111-
OSF_NIPYPE_URL: "https://files.osf.io/v1/resources/nefdp/providers/osfstorage"
112-
command: |
113-
export DATA_NIPYPE_TUTORIAL_URL="${OSF_NIPYPE_URL}/57f4739cb83f6901ed94bf21"
114-
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_TUTORIAL_URL" | tar xj
219+
name: Run SPM test workflow - 4D inputs (py36)
220+
no_output_timeout: 40m
221+
environment: *test_environment
222+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_linear_4d.sh
223+
- run: *_run_codecov_smoke
224+
- store_artifacts: *store_artifacts_kwds
115225

116-
export DATA_NIPYPE_FSL_COURSE="${OSF_NIPYPE_URL}/57f472cf9ad5a101f977ecfe"
117-
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_COURSE" | tar xz
226+
test_py3_fmri_spm_dartel_multiproc:
227+
machine: *machine_kwds
228+
working_directory: /home/circleci/nipype
229+
steps:
230+
- checkout:
231+
path: /home/circleci/nipype
232+
- attach_workspace:
233+
at: /tmp
234+
- run: *set_pr_number
235+
- run: *generate_dockerfiles
236+
- run: *modify_nipype_version
237+
- run: *get_base_image
238+
- run: *build_main_image_py36
239+
- run: *_get_codecov
240+
- run: *_download_test_data
241+
- run: *prepare_working_directory
242+
- run:
243+
name: Run SPM DARTEL Level 1 pipeline (py36)
244+
no_output_timeout: 1h
245+
environment: *test_environment
246+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l1.sh
247+
- run:
248+
name: Run SPM DARTEL Level 2 pipeline (py36)
249+
no_output_timeout: 30m
250+
environment: *test_environment
251+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_dartel_multiproc_l2.sh
252+
- run: *_run_codecov_smoke
253+
- store_artifacts: *store_artifacts_kwds
118254

119-
export DATA_NIPYPE_FSL_FEEDS="${OSF_NIPYPE_URL}/57f473066c613b01f113e7af"
120-
curl -sSL --retry 5 --connect-timeout 15 "$DATA_NIPYPE_FSL_FEEDS" | tar xz
255+
test_fmri_spm_nested_multiproc:
256+
machine: *machine_kwds
257+
working_directory: /home/circleci/nipype
258+
steps:
259+
- checkout:
260+
path: /home/circleci/nipype
261+
- attach_workspace:
262+
at: /tmp
263+
- run: *set_pr_number
264+
- run: *generate_dockerfiles
265+
- run: *modify_nipype_version
266+
- run: *get_base_image
267+
- run: *build_main_image_py36
268+
- run: *build_main_image_py27
269+
- run: *_get_codecov
270+
- run: *_download_test_data
271+
- run: *prepare_working_directory
121272
- run:
122-
name: Run tests
123-
no_output_timeout: 4h
124-
environment:
125-
WORKDIR: /home/circleci/work
126-
command: |
127-
mkdir -p "$WORKDIR"
128-
chmod -R 777 "$WORKDIR"
129-
bash /home/circleci/nipype/.circleci/tests.sh
273+
name: Run SPM Nested Level 1 pipeline (py36)
274+
no_output_timeout: 1h
275+
environment: *test_environment
276+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_fmri_spm_nested_multiproc_l1.sh
277+
- run:
278+
name: Run SPM Nested Level 2 pipeline (py27)
279+
no_output_timeout: 30m
280+
environment: *test_environment
281+
command: bash -ux /home/circleci/nipype/.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh
282+
- run: *_run_codecov_smoke
283+
- store_artifacts: *store_artifacts_kwds
284+
- run:
285+
name: Build docs (py36)
286+
no_output_timeout: 30m
287+
environment: *test_environment
288+
command: bash -ux /home/circleci/nipype/.circleci/test_py3_docs.sh
130289
- store_artifacts:
131-
path: /home/circleci/work/tests
290+
path: /home/circleci/work/docs
132291
- run:
133-
name: Save Docker images to workspace
292+
name: Save Docker images to workspace if on master
134293
no_output_timeout: 60m
135294
command: |
136-
if [ "$CIRCLE_NODE_INDEX" -eq "0" ] && [ "$CIRCLE_BRANCH" == "master" ]; then
295+
if [ "$CIRCLE_BRANCH" = "master" -a -z "$CIRCLE_PULL_REQUEST" ]; then
137296
docker save nipype/nipype:base \
138297
nipype/nipype:latest \
139-
nipype/nipype:py36 \
140-
nipype/nipype:py27 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz
141-
du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz
298+
nipype/nipype:py27 \
299+
nipype/nipype:py36 | gzip -1 > /tmp/docker/nipype-base-latest-py36-py27.tar.gz \
300+
&& du -h /tmp/docker/nipype-base-latest-py36-py27.tar.gz
142301
fi
143302
- persist_to_workspace:
144303
root: /tmp
145304
paths:
146-
- docker/*
147-
305+
- docker
148306

149307
deploy:
150308
docker:
@@ -180,15 +338,27 @@ jobs:
180338

181339
workflows:
182340
version: 2
183-
build_test_deply:
341+
build_test_deploy:
184342
jobs:
185343
- compare_base_dockerfiles
186-
- build_and_test:
344+
- test_pytest:
345+
requires:
346+
- compare_base_dockerfiles
347+
- test_py3_fmri_fsl_spm:
348+
requires:
349+
- compare_base_dockerfiles
350+
- test_py3_fmri_spm_dartel_multiproc:
351+
requires:
352+
- compare_base_dockerfiles
353+
- test_fmri_spm_nested_multiproc:
187354
requires:
188355
- compare_base_dockerfiles
189356
- deploy:
190357
filters:
191358
branches:
192359
only: master
193360
requires:
194-
- build_and_test
361+
- test_pytest
362+
- test_fmri_spm_nested_multiproc
363+
- test_py3_fmri_fsl_spm
364+
- test_py3_fmri_spm_dartel_multiproc

‎.circleci/test_py2_fmri_spm_nested_multiproc_l2.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
docker run --rm=false -t -v $WORKDIR:/work -v $HOME/examples:/data/examples:ro -w /work -e NIPYPE_NUMBER_OF_CPUS=4 -e NIPYPE_RESOURCE_MONITOR=1 "${DOCKER_IMAGE}:py27" /usr/bin/run_examples.sh fmri_spm_nested MultiProc /data/examples/ l2pipeline

0 commit comments

Comments
 (0)