From 447297ce597d2762db40afaa2349fc927c90f897 Mon Sep 17 00:00:00 2001 From: Thomas Lively <tlively@google.com> Date: Tue, 20 Jun 2017 13:37:58 -0700 Subject: [PATCH 1/5] Make wasm32 buildbot test LLVM backend This adds the experimental targets option to configure so it can be used by the builders and changes the wasm32 Dockerfile accordingly. Instead of using LLVM from the emsdk, the builder's emscripten tools now uses the Rust in-tree LLVM, since this is the one built with wasm support. --- configure | 1 + src/bootstrap/config.rs | 3 +++ src/ci/docker/disabled/wasm32/Dockerfile | 6 +++--- src/ci/docker/scripts/emscripten.sh | 5 ++++- src/tools/compiletest/src/runtest.rs | 11 ++++++++--- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/configure b/configure index 2b493ee91b50a..e08bcc028272e 100755 --- a/configure +++ b/configure @@ -490,6 +490,7 @@ valopt musl-root-armhf "" "arm-unknown-linux-musleabihf install directory" valopt musl-root-armv7 "" "armv7-unknown-linux-musleabihf install directory" valopt extra-filename "" "Additional data that is hashed and passed to the -C extra-filename flag" valopt qemu-armhf-rootfs "" "rootfs in qemu testing, you probably don't want to use this" +valopt experimental-targets "" "experimental LLVM targets to build" if [ -e ${CFG_SRC_DIR}.git ] then diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 902cd0997a8ed..3ada846e38236 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -497,6 +497,9 @@ impl Config { "CFG_TARGET" if value.len() > 0 => { self.target.extend(value.split(" ").map(|s| s.to_string())); } + "CFG_EXPERIMENTAL_TARGETS" if value.len() > 0 => { + self.llvm_experimental_targets = Some(value.to_string()); + } "CFG_MUSL_ROOT" if value.len() > 0 => { self.musl_root = Some(parse_configure_path(value)); } diff --git a/src/ci/docker/disabled/wasm32/Dockerfile b/src/ci/docker/disabled/wasm32/Dockerfile index daf398ac0962f..f3dd48ae411d7 100644 --- a/src/ci/docker/disabled/wasm32/Dockerfile +++ b/src/ci/docker/disabled/wasm32/Dockerfile @@ -20,7 +20,7 @@ RUN sh /scripts/dumb-init.sh # emscripten COPY scripts/emscripten.sh /scripts/ RUN bash /scripts/emscripten.sh -COPY wasm32/node.sh /usr/local/bin/node +COPY disabled/wasm32/node.sh /usr/local/bin/node # env ENV PATH=$PATH:/emsdk-portable @@ -30,9 +30,9 @@ ENV EMSCRIPTEN=/emsdk-portable/emscripten/1.37.13/ ENV BINARYEN_ROOT=/emsdk-portable/clang/e1.37.13_64bit/binaryen/ ENV EM_CONFIG=/emsdk-portable/.emscripten -ENV TARGETS=wasm32-unknown-emscripten +ENV TARGETS=wasm32-unknown-emscripten,wasm32-experimental-emscripten -ENV RUST_CONFIGURE_ARGS --target=$TARGETS +ENV RUST_CONFIGURE_ARGS --target=$TARGETS --experimental-targets=WebAssembly ENV SCRIPT python2.7 ../x.py test --target $TARGETS diff --git a/src/ci/docker/scripts/emscripten.sh b/src/ci/docker/scripts/emscripten.sh index 8aa5a98d7fc52..516ef0ae0d27a 100644 --- a/src/ci/docker/scripts/emscripten.sh +++ b/src/ci/docker/scripts/emscripten.sh @@ -40,9 +40,12 @@ hide_output ./emsdk install sdk-1.37.13-64bit source ./emsdk_env.sh echo "main(){}" > a.c HOME=/emsdk-portable/ emcc a.c -HOME=/emsdk-portable/ emcc -s BINARYEN=1 a.c +HOME=/emsdk-portable/ emcc -s WASM=1 a.c rm -f a.* +# Make emscripten use Rust's LLVM +echo "LLVM_ROOT='/checkout/obj/build/x86_64-unknown-linux-gnu/llvm/bin'" >> /root/.emscripten + # Make emsdk usable by any user cp /root/.emscripten /emsdk-portable chmod a+rxw -R /emsdk-portable diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 01419c4257074..3b3a94c6c5825 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -1280,6 +1280,12 @@ actual:\n\ let extra_link_args = vec!["-L".to_owned(), aux_dir.to_str().unwrap().to_owned()]; + let mut env = self.props.rustc_env.clone(); + // Tell emscripten to link using libc produced with LLVM backend + if self.config.target.contains("wasm32") && self.config.target.contains("experimental") { + env.push(("EMCC_WASM_BACKEND".to_string(), "1".to_string())); + } + for rel_ab in &self.props.aux_builds { let aux_testpaths = self.compute_aux_test_paths(rel_ab); let aux_props = self.props.from_aux_file(&aux_testpaths.file, @@ -1319,7 +1325,7 @@ actual:\n\ }; let aux_args = aux_cx.make_compile_args(crate_type, &aux_testpaths.file, aux_output); let auxres = aux_cx.compose_and_run(aux_args, - Vec::new(), + env.clone(), aux_cx.config.compile_lib_path.to_str().unwrap(), Some(aux_dir.to_str().unwrap()), None); @@ -1332,13 +1338,12 @@ actual:\n\ } self.compose_and_run(args, - self.props.rustc_env.clone(), + env, self.config.compile_lib_path.to_str().unwrap(), Some(aux_dir.to_str().unwrap()), input) } - fn compose_and_run(&self, ProcArgs{ args, prog }: ProcArgs, procenv: Vec<(String, String)> , From 16da303209dc7d4ec84b10322c9f7d34f72b2985 Mon Sep 17 00:00:00 2001 From: Thomas Lively <tlively@google.com> Date: Thu, 22 Jun 2017 15:16:54 -0700 Subject: [PATCH 2/5] Add target option for linker environment variables This is used in wasm32-experimental-emscripten to ensure that emscripten links against the libc bitcode files produced by the wasm LLVM backend, instead of using fastcomp. --- src/librustc_back/target/mod.rs | 4 ++++ .../target/wasm32_experimental_emscripten.rs | 1 + src/librustc_trans/back/link.rs | 3 +++ src/tools/compiletest/src/runtest.rs | 10 ++-------- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/librustc_back/target/mod.rs b/src/librustc_back/target/mod.rs index 37d6a6b95d937..5c95868471e43 100644 --- a/src/librustc_back/target/mod.rs +++ b/src/librustc_back/target/mod.rs @@ -282,6 +282,9 @@ pub struct TargetOptions { /// user-defined libraries. pub post_link_args: LinkArgs, + /// Environment variables to be set before invoking the linker. + pub link_env: Vec<(String, String)>, + /// Extra arguments to pass to the external assembler (when used) pub asm_args: Vec<String>, @@ -451,6 +454,7 @@ impl Default for TargetOptions { pre_link_objects_dll: Vec::new(), post_link_objects: Vec::new(), late_link_args: LinkArgs::new(), + link_env: Vec::new(), archive_format: "gnu".to_string(), custom_unwind_resume: false, lib_allocation_crate: "alloc_system".to_string(), diff --git a/src/librustc_back/target/wasm32_experimental_emscripten.rs b/src/librustc_back/target/wasm32_experimental_emscripten.rs index 1a95c93363adb..053fab5425019 100644 --- a/src/librustc_back/target/wasm32_experimental_emscripten.rs +++ b/src/librustc_back/target/wasm32_experimental_emscripten.rs @@ -30,6 +30,7 @@ pub fn target() -> Result<Target, String> { // possibly interpret the wasm, and a .wasm file exe_suffix: ".js".to_string(), linker_is_gnu: true, + link_env: vec![("EMCC_WASM_BACKEND".to_string(), "1".to_string())], allow_asm: false, obj_is_bitcode: true, is_like_emscripten: true, diff --git a/src/librustc_trans/back/link.rs b/src/librustc_trans/back/link.rs index 1f88f90dbbb28..a9af8b11f93ad 100644 --- a/src/librustc_trans/back/link.rs +++ b/src/librustc_trans/back/link.rs @@ -785,6 +785,9 @@ fn link_natively(sess: &Session, if let Some(args) = sess.target.target.options.post_link_args.get(&flavor) { cmd.args(args); } + for &(ref k, ref v) in &sess.target.target.options.link_env { + cmd.env(k, v); + } if sess.opts.debugging_opts.print_link_args { println!("{:?}", &cmd); diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 3b3a94c6c5825..0692e07253fbe 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -1280,12 +1280,6 @@ actual:\n\ let extra_link_args = vec!["-L".to_owned(), aux_dir.to_str().unwrap().to_owned()]; - let mut env = self.props.rustc_env.clone(); - // Tell emscripten to link using libc produced with LLVM backend - if self.config.target.contains("wasm32") && self.config.target.contains("experimental") { - env.push(("EMCC_WASM_BACKEND".to_string(), "1".to_string())); - } - for rel_ab in &self.props.aux_builds { let aux_testpaths = self.compute_aux_test_paths(rel_ab); let aux_props = self.props.from_aux_file(&aux_testpaths.file, @@ -1325,7 +1319,7 @@ actual:\n\ }; let aux_args = aux_cx.make_compile_args(crate_type, &aux_testpaths.file, aux_output); let auxres = aux_cx.compose_and_run(aux_args, - env.clone(), + Vec::new(), aux_cx.config.compile_lib_path.to_str().unwrap(), Some(aux_dir.to_str().unwrap()), None); @@ -1338,7 +1332,7 @@ actual:\n\ } self.compose_and_run(args, - env, + self.props.rustc_env.clone(), self.config.compile_lib_path.to_str().unwrap(), Some(aux_dir.to_str().unwrap()), input) From a4f97449b0a3043ecf91cdcc75ee69018cda6176 Mon Sep 17 00:00:00 2001 From: Thomas Lively <tlively@google.com> Date: Thu, 22 Jun 2017 18:04:20 -0700 Subject: [PATCH 3/5] Add wasm32-experimental-emscripten to wasm builder This modifies the builder to download and use the LLVM tools from the last known good build on the WebAssembly buildbot waterfall, since these tools are built with the WebAssembly LLVM backend enabled. --- src/ci/docker/disabled/wasm32/Dockerfile | 14 ++++++----- src/ci/docker/scripts/emscripten.sh | 32 +++++++++++++++--------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/ci/docker/disabled/wasm32/Dockerfile b/src/ci/docker/disabled/wasm32/Dockerfile index f3dd48ae411d7..025029c67456d 100644 --- a/src/ci/docker/disabled/wasm32/Dockerfile +++ b/src/ci/docker/disabled/wasm32/Dockerfile @@ -11,7 +11,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ sudo \ gdb \ - xz-utils + xz-utils \ + jq \ + bzip2 # dumb-init COPY scripts/dumb-init.sh /scripts/ @@ -22,6 +24,10 @@ COPY scripts/emscripten.sh /scripts/ RUN bash /scripts/emscripten.sh COPY disabled/wasm32/node.sh /usr/local/bin/node +# cache +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + # env ENV PATH=$PATH:/emsdk-portable ENV PATH=$PATH:/emsdk-portable/clang/e1.37.13_64bit/ @@ -32,13 +38,9 @@ ENV EM_CONFIG=/emsdk-portable/.emscripten ENV TARGETS=wasm32-unknown-emscripten,wasm32-experimental-emscripten -ENV RUST_CONFIGURE_ARGS --target=$TARGETS --experimental-targets=WebAssembly +ENV RUST_CONFIGURE_ARGS --target=$TARGETS ENV SCRIPT python2.7 ../x.py test --target $TARGETS -# cache -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - # init ENTRYPOINT ["/usr/bin/dumb-init", "--"] diff --git a/src/ci/docker/scripts/emscripten.sh b/src/ci/docker/scripts/emscripten.sh index 516ef0ae0d27a..e693f975f69bc 100644 --- a/src/ci/docker/scripts/emscripten.sh +++ b/src/ci/docker/scripts/emscripten.sh @@ -27,30 +27,38 @@ exit 1 set -x } +# Download emsdk cd / curl -L https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | \ tar -xz +# Download last known good emscripten from WebAssembly waterfall +BUILD=$(curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json | \ + jq '.build | tonumber') +curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/$BUILD/wasm-binaries.tbz2 | \ + hide_output tar xvkj + +# node 8 is required to run wasm +cd / +curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \ + tar -xJ + cd /emsdk-portable ./emsdk update hide_output ./emsdk install sdk-1.37.13-64bit ./emsdk activate sdk-1.37.13-64bit +# Make emscripten use wasm-ready node and LLVM tools +echo "NODE_JS='/node-v8.0.0-linux-x64/bin/node'" >> /root/.emscripten +echo "LLVM_ROOT='/wasm-install/bin'" >> /root/.emscripten + +# Make emsdk usable by any user +cp /root/.emscripten /emsdk-portable +chmod a+rxw -R /emsdk-portable + # Compile and cache libc source ./emsdk_env.sh echo "main(){}" > a.c HOME=/emsdk-portable/ emcc a.c HOME=/emsdk-portable/ emcc -s WASM=1 a.c rm -f a.* - -# Make emscripten use Rust's LLVM -echo "LLVM_ROOT='/checkout/obj/build/x86_64-unknown-linux-gnu/llvm/bin'" >> /root/.emscripten - -# Make emsdk usable by any user -cp /root/.emscripten /emsdk-portable -chmod a+rxw -R /emsdk-portable - -# node 8 is required to run wasm -cd / -curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \ - tar -xJ From 4ad6a9586050762a3d7e4e9bfee2faa36fbf726f Mon Sep 17 00:00:00 2001 From: Thomas Lively <tlively@google.com> Date: Fri, 23 Jun 2017 17:26:39 -0700 Subject: [PATCH 4/5] Add Target (de)serialization for environment vars Also turn WebAssembly backend back on in its builder. --- src/ci/docker/disabled/wasm32/Dockerfile | 2 +- src/librustc_back/target/mod.rs | 28 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/disabled/wasm32/Dockerfile b/src/ci/docker/disabled/wasm32/Dockerfile index 025029c67456d..ad57a1191e4e2 100644 --- a/src/ci/docker/disabled/wasm32/Dockerfile +++ b/src/ci/docker/disabled/wasm32/Dockerfile @@ -38,7 +38,7 @@ ENV EM_CONFIG=/emsdk-portable/.emscripten ENV TARGETS=wasm32-unknown-emscripten,wasm32-experimental-emscripten -ENV RUST_CONFIGURE_ARGS --target=$TARGETS +ENV RUST_CONFIGURE_ARGS --target=$TARGETS --experimental-targets=WebAssembly ENV SCRIPT python2.7 ../x.py test --target $TARGETS diff --git a/src/librustc_back/target/mod.rs b/src/librustc_back/target/mod.rs index 5c95868471e43..5df227e39acbd 100644 --- a/src/librustc_back/target/mod.rs +++ b/src/librustc_back/target/mod.rs @@ -624,6 +624,21 @@ impl Target { base.options.$key_name = args; } } ); + ($key_name:ident, env) => ( { + let name = (stringify!($key_name)).replace("_", "-"); + if let Some(a) = obj.find(&name[..]).and_then(|o| o.as_array()) { + for o in a { + if let Some(s) = o.as_string() { + let p = s.split('=').collect::<Vec<_>>(); + if p.len() == 2 { + let k = p[0].to_string(); + let v = p[1].to_string(); + base.options.$key_name.push((k, v)); + } + } + } + } + } ); } key!(is_builtin, bool); @@ -635,6 +650,7 @@ impl Target { key!(late_link_args, link_args); key!(post_link_objects, list); key!(post_link_args, link_args); + key!(link_env, env); key!(asm_args, list); key!(cpu); key!(features); @@ -789,6 +805,17 @@ impl ToJson for Target { d.insert(name.to_string(), obj.to_json()); } } ); + (env - $attr:ident) => ( { + let name = (stringify!($attr)).replace("_", "-"); + if default.$attr != self.options.$attr { + let obj = self.options.$attr + .iter() + .map(|&(ref k, ref v)| k.clone() + "=" + &v) + .collect::<Vec<_>>(); + d.insert(name.to_string(), obj.to_json()); + } + } ); + } target_val!(llvm_target); @@ -810,6 +837,7 @@ impl ToJson for Target { target_option_val!(link_args - late_link_args); target_option_val!(post_link_objects); target_option_val!(link_args - post_link_args); + target_option_val!(env - link_env); target_option_val!(asm_args); target_option_val!(cpu); target_option_val!(features); From c130b83bae47777e8a9e1b4cb3d6249cf987c0d6 Mon Sep 17 00:00:00 2001 From: Thomas Lively <tlively@google.com> Date: Sat, 24 Jun 2017 11:35:48 -0700 Subject: [PATCH 5/5] Restore old emscripten.sh for use by asmjs --- src/ci/docker/disabled/wasm32/Dockerfile | 4 +- src/ci/docker/scripts/emscripten-wasm.sh | 64 ++++++++++++++++++++++++ src/ci/docker/scripts/emscripten.sh | 31 ++++-------- 3 files changed, 76 insertions(+), 23 deletions(-) create mode 100644 src/ci/docker/scripts/emscripten-wasm.sh diff --git a/src/ci/docker/disabled/wasm32/Dockerfile b/src/ci/docker/disabled/wasm32/Dockerfile index ad57a1191e4e2..c75b5d455c522 100644 --- a/src/ci/docker/disabled/wasm32/Dockerfile +++ b/src/ci/docker/disabled/wasm32/Dockerfile @@ -20,8 +20,8 @@ COPY scripts/dumb-init.sh /scripts/ RUN sh /scripts/dumb-init.sh # emscripten -COPY scripts/emscripten.sh /scripts/ -RUN bash /scripts/emscripten.sh +COPY scripts/emscripten-wasm.sh /scripts/ +RUN bash /scripts/emscripten-wasm.sh COPY disabled/wasm32/node.sh /usr/local/bin/node # cache diff --git a/src/ci/docker/scripts/emscripten-wasm.sh b/src/ci/docker/scripts/emscripten-wasm.sh new file mode 100644 index 0000000000000..e693f975f69bc --- /dev/null +++ b/src/ci/docker/scripts/emscripten-wasm.sh @@ -0,0 +1,64 @@ +# Copyright 2017 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -ex + +hide_output() { + set +x + on_err=" +echo ERROR: An error was encountered with the build. +cat /tmp/build.log +exit 1 +" + trap "$on_err" ERR + bash -c "while true; do sleep 30; echo \$(date) - building ...; done" & + PING_LOOP_PID=$! + $@ &> /tmp/build.log + trap - ERR + kill $PING_LOOP_PID + rm -f /tmp/build.log + set -x +} + +# Download emsdk +cd / +curl -L https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | \ + tar -xz + +# Download last known good emscripten from WebAssembly waterfall +BUILD=$(curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json | \ + jq '.build | tonumber') +curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/$BUILD/wasm-binaries.tbz2 | \ + hide_output tar xvkj + +# node 8 is required to run wasm +cd / +curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \ + tar -xJ + +cd /emsdk-portable +./emsdk update +hide_output ./emsdk install sdk-1.37.13-64bit +./emsdk activate sdk-1.37.13-64bit + +# Make emscripten use wasm-ready node and LLVM tools +echo "NODE_JS='/node-v8.0.0-linux-x64/bin/node'" >> /root/.emscripten +echo "LLVM_ROOT='/wasm-install/bin'" >> /root/.emscripten + +# Make emsdk usable by any user +cp /root/.emscripten /emsdk-portable +chmod a+rxw -R /emsdk-portable + +# Compile and cache libc +source ./emsdk_env.sh +echo "main(){}" > a.c +HOME=/emsdk-portable/ emcc a.c +HOME=/emsdk-portable/ emcc -s WASM=1 a.c +rm -f a.* diff --git a/src/ci/docker/scripts/emscripten.sh b/src/ci/docker/scripts/emscripten.sh index e693f975f69bc..cf5eecbdb6c8c 100644 --- a/src/ci/docker/scripts/emscripten.sh +++ b/src/ci/docker/scripts/emscripten.sh @@ -27,38 +27,27 @@ exit 1 set -x } -# Download emsdk cd / curl -L https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | \ tar -xz -# Download last known good emscripten from WebAssembly waterfall -BUILD=$(curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json | \ - jq '.build | tonumber') -curl -L https://storage.googleapis.com/wasm-llvm/builds/linux/$BUILD/wasm-binaries.tbz2 | \ - hide_output tar xvkj - -# node 8 is required to run wasm -cd / -curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \ - tar -xJ - cd /emsdk-portable ./emsdk update hide_output ./emsdk install sdk-1.37.13-64bit ./emsdk activate sdk-1.37.13-64bit -# Make emscripten use wasm-ready node and LLVM tools -echo "NODE_JS='/node-v8.0.0-linux-x64/bin/node'" >> /root/.emscripten -echo "LLVM_ROOT='/wasm-install/bin'" >> /root/.emscripten - -# Make emsdk usable by any user -cp /root/.emscripten /emsdk-portable -chmod a+rxw -R /emsdk-portable - # Compile and cache libc source ./emsdk_env.sh echo "main(){}" > a.c HOME=/emsdk-portable/ emcc a.c -HOME=/emsdk-portable/ emcc -s WASM=1 a.c +HOME=/emsdk-portable/ emcc -s BINARYEN=1 a.c rm -f a.* + +# Make emsdk usable by any user +cp /root/.emscripten /emsdk-portable +chmod a+rxw -R /emsdk-portable + +# node 8 is required to run wasm +cd / +curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \ + tar -xJ