Skip to content

Commit d6f721d

Browse files
jeffdonahuejeffdonahue
committed
Merge pull request BVLC#943 from jeffdonahue/parallel-travis-builds
add Travis build matrix to do parallel builds for (make, CMake) x (with CUDA, without CUDA)
2 parents 0db54c4 + bff1809 commit d6f721d

5 files changed

+146
-57
lines changed

‎.travis.yml

+15-57
Original file line numberDiff line numberDiff line change
@@ -1,76 +1,34 @@
1+
# Use a build matrix to do two builds in parallel:
2+
# one using CMake, and one using make.
3+
env:
4+
matrix:
5+
- WITH_CUDA=false WITH_CMAKE=false
6+
- WITH_CUDA=false WITH_CMAKE=true
7+
- WITH_CUDA=true WITH_CMAKE=false
8+
- WITH_CUDA=true WITH_CMAKE=true
9+
110
language: cpp
211

312
# Cache Ubuntu apt packages.
413
cache: apt
514

6-
compiler:
7-
- gcc
8-
# Disable clang build: doesn't seem to work on Linux.
9-
# (@jeffdonahue: Travis buildbot's failure behavior is similar to what I see
10-
# building on Linux.)
11-
# - clang
15+
compiler: gcc
1216

1317
before_install:
14-
- echo $LANG
15-
- echo $LC_ALL
1618
- export NUM_THREADS=4
1719
- alias make="make --jobs=$NUM_THREADS"
18-
- sudo add-apt-repository ppa:ubuntu-sdk-team/ppa -y
19-
- sudo apt-get -y update
20-
- sudo apt-get -y install wget git curl python-dev python-numpy libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libboost-dev libboost-system-dev libboost-python-dev libhdf5-serial-dev protobuf-compiler libatlas-dev libatlas-base-dev bc cmake
20+
- export SCRIPTS=./scripts/travis
21+
- set -e # fail when a script fails
2122

2223
install:
23-
- wget https://google-glog.googlecode.com/files/glog-0.3.3.tar.gz -O /tmp/glog-0.3.3.tar.gz && tar -C /tmp -xzvf /tmp/glog-0.3.3.tar.gz && rm /tmp/glog-0.3.3.tar.gz
24-
- cd /tmp/glog-0.3.3 && ./configure && make && sudo make install && cd -
25-
- wget https://github.com/schuhschuh/gflags/archive/master.zip -O /tmp/gflags-master.zip && pushd /tmp/ && unzip gflags-master.zip && cd gflags-master && mkdir build && cd build && export CXXFLAGS="-fPIC" && cmake .. && make VERBOSE=1 && sudo make install && popd
26-
- curl http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1204/x86_64/cuda-repo-ubuntu1204_6.0-37_amd64.deb -o /tmp/cuda_install.deb && sudo dpkg -i /tmp/cuda_install.deb && rm /tmp/cuda_install.deb
27-
- sudo apt-get -y update
28-
# Install the minimal CUDA subpackages required to test Caffe build.
29-
# For a full CUDA installation, add 'cuda' to the list of packages.
30-
- sudo apt-get -y install cuda-core-6-0 cuda-extra-libs-6-0
31-
# Create CUDA symlink at /usr/local/cuda
32-
# (This would normally be created by the CUDA installer, but we create it
33-
# manually since we did a partial installation.)
34-
- sudo ln -s /usr/local/cuda-6.0 /usr/local/cuda
35-
- curl https://gitorious.org/mdb/mdb/archive/7f038d0f15bec57b4c07aa3f31cd5564c88a1897.tar.gz -o /tmp/mdb.tar.gz && tar -C /tmp -xzvf /tmp/mdb.tar.gz && rm /tmp/mdb.tar.gz
36-
- cd /tmp/mdb-mdb/libraries/liblmdb/ && make && sudo make install && cd -
24+
- source $SCRIPTS/travis_install.sh
3725

3826
before_script:
39-
- mv Makefile.config.example Makefile.config
4027
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
28+
- if ! $WITH_CMAKE; then source $SCRIPTS/travis_setup_makefile_config.sh; fi
4129

4230
script:
43-
# CMake build
44-
- mkdir build
45-
- cd build
46-
## CPU-GPU: build only
47-
- cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release ..
48-
- make --keep-going
49-
- make clean && rm -rf *
50-
## CPU-only: comprehensive
51-
- cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=ON ..
52-
- make --keep-going
53-
- make runtest
54-
- make lint
55-
- make clean
56-
## Cleanup CMake build
57-
- cd ..
58-
- rm -rf build
59-
60-
# Make build
61-
## CPU-GPU: build only
62-
- export CPU_ONLY=0
63-
- make --keep-going all
64-
- make clean
65-
## CPU-only: comprehensive
66-
- export CPU_ONLY=1
67-
- make --keep-going all test warn lint
68-
- make runtest
69-
- make all
70-
- make test
71-
- make warn
72-
- make lint
73-
- make pycaffe
31+
- if $WITH_CUDA; then source $SCRIPTS/travis_build.sh; else source $SCRIPTS/travis_build_and_test.sh; fi
7432

7533
notifications:
7634
# Emails are sent to the committer's git-configured email address by default,

‎scripts/travis/travis_build.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
# Script called by Travis to do a full build of Caffe,
3+
# including CUDA functionality.
4+
5+
if $WITH_CMAKE; then
6+
mkdir build
7+
cd build
8+
cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release ..
9+
make --keep-going
10+
cd -
11+
else
12+
export CPU_ONLY=0
13+
make --keep-going all test pycaffe warn
14+
make all
15+
make test
16+
make pycaffe
17+
make warn
18+
fi

‎scripts/travis/travis_build_and_test.sh

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
# Script called by Travis to do a CPU-only build of and test Caffe.
3+
4+
if $WITH_CMAKE; then
5+
mkdir build
6+
cd build
7+
cmake -DBUILD_PYTHON=ON -DBUILD_EXAMPLES=ON -DCMAKE_BUILD_TYPE=Release -DCPU_ONLY=ON ..
8+
make --keep-going
9+
make runtest
10+
make lint
11+
make clean
12+
cd -
13+
else
14+
export CPU_ONLY=1
15+
make --keep-going all test pycaffe warn lint
16+
make runtest
17+
make all
18+
make test
19+
make pycaffe
20+
make warn
21+
make lint
22+
fi

‎scripts/travis/travis_install.sh

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/bin/bash
2+
3+
# Install apt packages where the Ubuntu 12.04 default works for Caffe
4+
sudo apt-get -y update
5+
sudo apt-get install \
6+
wget git curl \
7+
python-dev python-numpy \
8+
libleveldb-dev libsnappy-dev libopencv-dev \
9+
libboost-dev libboost-system-dev libboost-python-dev \
10+
libprotobuf-dev protobuf-compiler \
11+
libatlas-dev libatlas-base-dev \
12+
libhdf5-serial-dev \
13+
bc
14+
15+
# Add a special apt-repository to install CMake 2.8.9 for CMake Caffe build,
16+
# if needed. By default, Aptitude in Ubuntu 12.04 installs CMake 2.8.7, but
17+
# Caffe requires a minimum CMake version of 2.8.8.
18+
if $WITH_CMAKE; then
19+
sudo add-apt-repository -y ppa:ubuntu-sdk-team/ppa
20+
sudo apt-get -y update
21+
sudo apt-get -y install cmake || (echo "CMake install failed"; exit 1)
22+
fi
23+
24+
# Install glog
25+
GLOG_URL=https://google-glog.googlecode.com/files/glog-0.3.3.tar.gz
26+
GLOG_FILE=/tmp/glog-0.3.3.tar.gz
27+
pushd .
28+
wget $GLOG_URL -O $GLOG_FILE && \
29+
tar -C /tmp -xzvf $GLOG_FILE && \
30+
rm $GLOG_FILE && \
31+
cd /tmp/glog-0.3.3 && \
32+
./configure && make && sudo make install || \
33+
(echo "glog install failed"; exit 1)
34+
popd
35+
36+
# Install gflags
37+
GFLAGS_URL=https://github.com/schuhschuh/gflags/archive/master.zip
38+
GFLAGS_FILE=/tmp/gflags-master.zip
39+
pushd .
40+
wget $GFLAGS_URL -O $GFLAGS_FILE && \
41+
cd /tmp/ && unzip gflags-master.zip && \
42+
cd gflags-master && \
43+
mkdir build && \
44+
cd build && \
45+
export CXXFLAGS="-fPIC" && \
46+
cmake .. && make VERBOSE=1 && sudo make install || \
47+
(echo "gflags install failed"; exit 1)
48+
popd
49+
50+
# Install CUDA, if needed
51+
CUDA_URL=http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1204/x86_64/cuda-repo-ubuntu1204_6.0-37_amd64.deb
52+
CUDA_FILE=/tmp/cuda_install.deb
53+
if $WITH_CUDA; then
54+
curl $CUDA_URL -o $CUDA_FILE && \
55+
sudo dpkg -i $CUDA_FILE ||
56+
(echo "CUDA install failed"; exit 1)
57+
rm -f $CUDA_FILE
58+
sudo apt-get -y update
59+
# Install the minimal CUDA subpackages required to test Caffe build.
60+
# For a full CUDA installation, add 'cuda' to the list of packages.
61+
sudo apt-get -y install cuda-core-6-0 cuda-extra-libs-6-0
62+
# Create CUDA symlink at /usr/local/cuda
63+
# (This would normally be created by the CUDA installer, but we create it
64+
# manually since we did a partial installation.)
65+
sudo ln -s /usr/local/cuda-6.0 /usr/local/cuda ||
66+
(echo "CUDA symlink creation failed"; exit 1)
67+
fi
68+
69+
# Install LMDB
70+
LMDB_URL=https://gitorious.org/mdb/mdb/archive/7f038d0f15bec57b4c07aa3f31cd5564c88a1897.tar.gz
71+
LMDB_FILE=/tmp/mdb.tar.gz
72+
pushd .
73+
curl $LMDB_URL -o $LMDB_FILE && \
74+
tar -C /tmp -xzvf $LMDB_FILE && \
75+
cd /tmp/mdb-mdb/libraries/liblmdb/ && \
76+
make && sudo make install || \
77+
(echo "LMDB install failed"; exit 1)
78+
popd
79+
rm -f $LMDB_FILE

‎scripts/travis/travis_setup_makefile_config.sh

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
mv Makefile.config.example Makefile.config
4+
5+
if $WITH_CUDA; then
6+
# Remove default gencode set; only generate compute_50.
7+
sed -i 's/-gencode arch=.*\\//' Makefile.config
8+
sed -i 's/CUDA_ARCH :=//' Makefile.config
9+
GENCODE="-gencode arch=compute_50,code=sm_50"
10+
GENCODE="$GENCODE -gencode arch=compute_50,code=compute_50"
11+
echo "CUDA_ARCH := $GENCODE" >> Makefile.config
12+
fi

0 commit comments

Comments
 (0)