Skip to content

Commit 447bcd3

Browse files
George Gensurefacebook-github-bot
George Gensure
authored and
facebook-github-bot
committedApr 7, 2020
Bazel build of pytorch with gating CI (pytorch#36011)
Summary: Pull Request resolved: pytorch#36011 Differential Revision: D20873430 Pulled By: malfet fbshipit-source-id: 8ffffd10ca0ff8bdab578a70a9b2b777aed985d0
1 parent 64594d8 commit 447bcd3

37 files changed

+4516
-53
lines changed
 

‎.bazelrc

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
build --copt=--std=c++14
2+
build --copt=-I.
3+
build --copt=-isystem --copt bazel-out/k8-fastbuild/bin

‎.bazelversion

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
2.2.0

‎.circleci/cimodel/data/pytorch_build_definitions.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,9 @@ def instantiate_configs():
187187

188188
root = get_root()
189189
found_configs = conf_tree.dfs(root)
190-
restrict_phases = None
191190
for fc in found_configs:
192191

192+
restrict_phases = None
193193
distro_name = fc.find_prop("distro_name")
194194
compiler_name = fc.find_prop("compiler_name")
195195
compiler_version = fc.find_prop("compiler_version")

‎.circleci/config.yml

+85
Original file line numberDiff line numberDiff line change
@@ -1557,6 +1557,77 @@ jobs:
15571557
cd ${PROJ_ROOT}/ios/TestApp
15581558
instruments -s -devices
15591559
fastlane scan
1560+
pytorch_linux_bazel_build:
1561+
<<: *pytorch_params
1562+
machine:
1563+
image: ubuntu-1604:201903-01
1564+
steps:
1565+
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
1566+
- attach_scripts
1567+
- setup_linux_system_environment
1568+
- checkout
1569+
- setup_ci_environment
1570+
- run:
1571+
name: Bazel Build
1572+
no_output_timeout: "1h"
1573+
command: |
1574+
set -e
1575+
# Pull Docker image and run build
1576+
echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
1577+
time docker pull ${DOCKER_IMAGE} >/dev/null
1578+
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
1579+
1580+
echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
1581+
1582+
git submodule sync && git submodule update -q --init --recursive
1583+
1584+
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
1585+
1586+
export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
1587+
1588+
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
1589+
1590+
# Push intermediate Docker image for next phase to use
1591+
if [ -z "${BUILD_ONLY}" ]; then
1592+
# Augment our output image name with bazel to avoid collisions
1593+
output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
1594+
export COMMIT_DOCKER_IMAGE=$output_image
1595+
docker commit "$id" ${COMMIT_DOCKER_IMAGE}
1596+
time docker push ${COMMIT_DOCKER_IMAGE}
1597+
fi
1598+
1599+
pytorch_linux_bazel_test:
1600+
<<: *pytorch_params
1601+
machine:
1602+
image: ubuntu-1604:201903-01
1603+
steps:
1604+
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
1605+
- attach_scripts
1606+
- setup_linux_system_environment
1607+
- setup_ci_environment
1608+
- run:
1609+
name: Test
1610+
no_output_timeout: "90m"
1611+
command: |
1612+
set -e
1613+
output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
1614+
export COMMIT_DOCKER_IMAGE=$output_image
1615+
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
1616+
1617+
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
1618+
1619+
if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
1620+
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
1621+
else
1622+
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
1623+
fi
1624+
1625+
if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
1626+
export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
1627+
else
1628+
export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
1629+
fi
1630+
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
15601631
15611632
promote_s3:
15621633
<<: *promote_common
@@ -2438,6 +2509,20 @@ workflows:
24382509
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7:e43973a9-9d5a-4138-9181-a08a0fc55e2f"
24392510
use_cuda_docker_runtime: "1"
24402511
resource_class: gpu.medium
2512+
- pytorch_linux_bazel_build:
2513+
name: pytorch_bazel_build
2514+
requires:
2515+
- setup
2516+
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-build"
2517+
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
2518+
resource_class: large
2519+
- pytorch_linux_bazel_test:
2520+
name: pytorch_bazel_test
2521+
requires:
2522+
- setup
2523+
- pytorch_bazel_build
2524+
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-test"
2525+
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
24412526
- caffe2_linux_build:
24422527
name: caffe2_onnx_main_py3_6_clang7_ubuntu16_04_build
24432528
requires:

‎.circleci/generate_config_yml.py

+1
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ def write(self, output_filehandle):
101101
File("workflows-pytorch-ios-builds.yml"),
102102
File("workflows-pytorch-mobile-builds.yml"),
103103
File("workflows-pytorch-ge-config-tests.yml"),
104+
File("workflows-pytorch-bazel-builds.yml"),
104105
Listgen(caffe2_build_definitions.get_workflow_jobs, 3),
105106
File("workflows-binary-builds-smoke-subset.yml"),
106107
Listgen(binary_build_definitions.get_binary_smoke_test_jobs, 3),

‎.circleci/verbatim-sources/job-specs-custom.yml

+71
Original file line numberDiff line numberDiff line change
@@ -440,3 +440,74 @@
440440
cd ${PROJ_ROOT}/ios/TestApp
441441
instruments -s -devices
442442
fastlane scan
443+
pytorch_linux_bazel_build:
444+
<<: *pytorch_params
445+
machine:
446+
image: ubuntu-1604:201903-01
447+
steps:
448+
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
449+
- attach_scripts
450+
- setup_linux_system_environment
451+
- checkout
452+
- setup_ci_environment
453+
- run:
454+
name: Bazel Build
455+
no_output_timeout: "1h"
456+
command: |
457+
set -e
458+
# Pull Docker image and run build
459+
echo "DOCKER_IMAGE: "${DOCKER_IMAGE}
460+
time docker pull ${DOCKER_IMAGE} >/dev/null
461+
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
462+
463+
echo "Do NOT merge master branch into $CIRCLE_BRANCH in environment $BUILD_ENVIRONMENT"
464+
465+
git submodule sync && git submodule update -q --init --recursive
466+
467+
docker cp /home/circleci/project/. $id:/var/lib/jenkins/workspace
468+
469+
export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/build.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
470+
471+
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
472+
473+
# Push intermediate Docker image for next phase to use
474+
if [ -z "${BUILD_ONLY}" ]; then
475+
# Augment our output image name with bazel to avoid collisions
476+
output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
477+
export COMMIT_DOCKER_IMAGE=$output_image
478+
docker commit "$id" ${COMMIT_DOCKER_IMAGE}
479+
time docker push ${COMMIT_DOCKER_IMAGE}
480+
fi
481+
482+
pytorch_linux_bazel_test:
483+
<<: *pytorch_params
484+
machine:
485+
image: ubuntu-1604:201903-01
486+
steps:
487+
# See Note [Workspace for CircleCI scripts] in job-specs-setup.yml
488+
- attach_scripts
489+
- setup_linux_system_environment
490+
- setup_ci_environment
491+
- run:
492+
name: Test
493+
no_output_timeout: "90m"
494+
command: |
495+
set -e
496+
output_image=${DOCKER_IMAGE}-bazel-${CIRCLE_SHA1}
497+
export COMMIT_DOCKER_IMAGE=$output_image
498+
echo "DOCKER_IMAGE: "${COMMIT_DOCKER_IMAGE}
499+
500+
time docker pull ${COMMIT_DOCKER_IMAGE} >/dev/null
501+
502+
if [ -n "${USE_CUDA_DOCKER_RUNTIME}" ]; then
503+
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --runtime=nvidia -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
504+
else
505+
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${COMMIT_DOCKER_IMAGE})
506+
fi
507+
508+
if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
509+
export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/multigpu-test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
510+
else
511+
export COMMAND='((echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export CIRCLE_PULL_REQUEST=${CIRCLE_PULL_REQUEST}" && echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && .jenkins/pytorch/test.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
512+
fi
513+
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts

‎.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
- pytorch_linux_bazel_build:
2+
name: pytorch_bazel_build
3+
requires:
4+
- setup
5+
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-build"
6+
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"
7+
resource_class: large
8+
- pytorch_linux_bazel_test:
9+
name: pytorch_bazel_test
10+
requires:
11+
- setup
12+
- pytorch_bazel_build
13+
build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-test"
14+
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026"

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -255,3 +255,6 @@ TAGS
255255

256256
# clangd background index
257257
.clangd/
258+
259+
# bazel symlinks
260+
bazel-*

‎.jenkins/pytorch/build.sh

+57-50
Original file line numberDiff line numberDiff line change
@@ -180,65 +180,72 @@ if [[ "${BUILD_ENVIRONMENT}" == *clang* ]]; then
180180
export CXX=clang++
181181
fi
182182

183+
if [[ "$BUILD_ENVIRONMENT" == *-bazel-* ]]; then
184+
set -e
183185

184-
# check that setup.py would fail with bad arguments
185-
echo "The next three invocations are expected to fail with invalid command error messages."
186-
( ! get_exit_code python setup.py bad_argument )
187-
( ! get_exit_code python setup.py clean] )
188-
( ! get_exit_code python setup.py clean bad_argument )
186+
get_bazel
189187

190-
if [[ "$BUILD_ENVIRONMENT" != *libtorch* ]]; then
191-
192-
# ppc64le build fails when WERROR=1
193-
# set only when building other architectures
194-
# only use for "python setup.py install" line
195-
if [[ "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *clang* ]]; then
196-
WERROR=1 python setup.py install
197-
else
198-
python setup.py install
199-
fi
188+
tools/bazel build :torch
189+
else
190+
# check that setup.py would fail with bad arguments
191+
echo "The next three invocations are expected to fail with invalid command error messages."
192+
( ! get_exit_code python setup.py bad_argument )
193+
( ! get_exit_code python setup.py clean] )
194+
( ! get_exit_code python setup.py clean bad_argument )
195+
196+
if [[ "$BUILD_ENVIRONMENT" != *libtorch* ]]; then
197+
198+
# ppc64le build fails when WERROR=1
199+
# set only when building other architectures
200+
# only use for "python setup.py install" line
201+
if [[ "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *clang* ]]; then
202+
WERROR=1 python setup.py install
203+
else
204+
python setup.py install
205+
fi
200206

201-
# TODO: I'm not sure why, but somehow we lose verbose commands
202-
set -x
207+
# TODO: I'm not sure why, but somehow we lose verbose commands
208+
set -x
203209

204-
if which sccache > /dev/null; then
205-
echo 'PyTorch Build Statistics'
206-
sccache --show-stats
207-
fi
210+
if which sccache > /dev/null; then
211+
echo 'PyTorch Build Statistics'
212+
sccache --show-stats
213+
fi
208214

209-
assert_git_not_dirty
215+
assert_git_not_dirty
210216

211-
# Build custom operator tests.
212-
CUSTOM_OP_BUILD="$PWD/../custom-op-build"
213-
CUSTOM_OP_TEST="$PWD/test/custom_operator"
214-
python --version
215-
SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
216-
mkdir "$CUSTOM_OP_BUILD"
217-
pushd "$CUSTOM_OP_BUILD"
218-
cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)"
219-
make VERBOSE=1
220-
popd
221-
assert_git_not_dirty
222-
else
223-
# Test standalone c10 build
224-
if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda10.1-cudnn7-py3* ]]; then
225-
mkdir -p c10/build
226-
pushd c10/build
227-
cmake ..
228-
make -j
217+
# Build custom operator tests.
218+
CUSTOM_OP_BUILD="$PWD/../custom-op-build"
219+
CUSTOM_OP_TEST="$PWD/test/custom_operator"
220+
python --version
221+
SITE_PACKAGES="$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')"
222+
mkdir "$CUSTOM_OP_BUILD"
223+
pushd "$CUSTOM_OP_BUILD"
224+
cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES/torch" -DPYTHON_EXECUTABLE="$(which python)"
225+
make VERBOSE=1
229226
popd
230227
assert_git_not_dirty
231-
fi
228+
else
229+
# Test standalone c10 build
230+
if [[ "$BUILD_ENVIRONMENT" == *xenial-cuda10.1-cudnn7-py3* ]]; then
231+
mkdir -p c10/build
232+
pushd c10/build
233+
cmake ..
234+
make -j
235+
popd
236+
assert_git_not_dirty
237+
fi
232238

233-
# Test no-Python build
234-
echo "Building libtorch"
235-
# NB: Install outside of source directory (at the same level as the root
236-
# pytorch folder) so that it doesn't get cleaned away prior to docker push.
237-
BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
238-
mkdir -p ../cpp-build/caffe2
239-
pushd ../cpp-build/caffe2
240-
WERROR=1 VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
241-
popd
239+
# Test no-Python build
240+
echo "Building libtorch"
241+
# NB: Install outside of source directory (at the same level as the root
242+
# pytorch folder) so that it doesn't get cleaned away prior to docker push.
243+
BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
244+
mkdir -p ../cpp-build/caffe2
245+
pushd ../cpp-build/caffe2
246+
WERROR=1 VERBOSE=1 DEBUG=1 python $BUILD_LIBTORCH_PY
247+
popd
248+
fi
242249
fi
243250

244251
# Test XLA build

‎.jenkins/pytorch/common.sh

+9
Original file line numberDiff line numberDiff line change
@@ -187,3 +187,12 @@ function file_diff_from_base() {
187187
set -e
188188
git diff --name-only "$(git merge-base origin master HEAD)" > "$1"
189189
}
190+
191+
function get_bazel() {
192+
# download bazel version
193+
wget https://github.com/bazelbuild/bazel/releases/download/2.2.0/bazel-2.2.0-linux-x86_64 -O tools/bazel
194+
# verify content
195+
echo 'b2f002ea0e6194a181af6ac84cd94bd8dc797722eb2354690bebac92dda233ff tools/bazel' | sha256sum --quiet -c
196+
197+
chmod +x tools/bazel
198+
}

‎.jenkins/pytorch/test.sh

+12-2
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
4242
fi
4343

4444
# --user breaks ppc64le builds and these packages are already in ppc64le docker
45-
if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
45+
if [[ "$BUILD_ENVIRONMENT" != *ppc64le* ]] && [[ "$BUILD_ENVIRONMENT" != *-bazel-* ]] ; then
4646
# JIT C++ extensions require ninja.
4747
pip_install --user ninja
4848
# ninja is installed in /var/lib/jenkins/.local/bin
@@ -252,7 +252,15 @@ test_backward_compatibility() {
252252
assert_git_not_dirty
253253
}
254254

255-
if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* ]]; then
255+
test_bazel() {
256+
set -e
257+
258+
get_bazel
259+
260+
tools/bazel test --test_tag_filters=-gpu-required --test_filter=-*_CUDA :all_tests
261+
}
262+
263+
if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* || "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
256264
(cd test && python -c "import torch; print(torch.__config__.show())")
257265
(cd test && python -c "import torch; print(torch.__config__.parallel_info())")
258266
fi
@@ -278,6 +286,8 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-test2 || "${JOB_BASE_NAME}" == *-test2 ]]; t
278286
test_aten
279287
test_libtorch
280288
test_custom_script_ops
289+
elif [[ "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
290+
test_bazel
281291
else
282292
test_torchvision
283293
test_python_nn

0 commit comments

Comments
 (0)