Skip to content

Commit a8cd851

Browse files
abrownjohn-sharratt
abrown
authored and
john-sharratt
committedMar 5, 2023
test: run a subset of tests using libc-test (WebAssembly#346)
* test: run a subset of tests from `libc-test` This change introduces a `test` directory that retrieves the `libc-test` suite, compiles a subset of the tests using `wasi-libc`, and runs them with Wasmtime. * ci: run tests during CI This change includes some fixups to the filesystem to place Clang's runtime library for `wasm32-wasi` in the right location. Note that this CI action is limited to a single OS--Linux.
1 parent 5ec38e7 commit a8cd851

File tree

5 files changed

+202
-1
lines changed

5 files changed

+202
-1
lines changed
 

‎.github/workflows/main.yml

+17
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ jobs:
5656
curl -sSfL https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz | tar xJf -
5757
export CLANG_DIR=`pwd`/clang+llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04/bin
5858
echo "$CLANG_DIR" >> $GITHUB_PATH
59+
echo "CLANG_DIR=$CLANG_DIR" >> $GITHUB_ENV
5960
echo "CC=$CLANG_DIR/clang" >> $GITHUB_ENV
6061
echo "AR=$CLANG_DIR/llvm-ar" >> $GITHUB_ENV
6162
echo "NM=$CLANG_DIR/llvm-nm" >> $GITHUB_ENV
@@ -65,6 +66,22 @@ jobs:
6566
shell: bash
6667
run: make -j4
6768

69+
- name: Test
70+
shell: bash
71+
# For Clang linking to work correctly, we need to place Clang's runtime
72+
# library for `wasm32-wasi` in the right location (i.e., the `mkdir` and
73+
# `cp` below).
74+
run: |
75+
cd test
76+
make download
77+
export WASI_DIR=$(realpath $CLANG_DIR/../lib/clang/${{ matrix.clang_version }}/lib/wasi/)
78+
mkdir -p $WASI_DIR
79+
cp download/lib/wasi/libclang_rt.builtins-wasm32.a $WASI_DIR
80+
make test
81+
# The older version of Clang does not provide the expected symbol for the
82+
# test entrypoints: `undefined symbol: __main_argc_argv`.
83+
if: matrix.os == 'ubuntu-latest' && matrix.clang_version != '10.0.0'
84+
6885
- uses: actions/upload-artifact@v1
6986
with:
7087
# Upload the sysroot folder. Give it a name according to the OS it was built for.

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ THREAD_MODEL ?= posix
1919
MALLOC_IMPL ?= dlmalloc
2020
# yes or no
2121
BUILD_LIBC_TOP_HALF ?= yes
22-
# The directory where we're store intermediate artifacts.
22+
# The directory where we will store intermediate artifacts.
2323
OBJDIR ?= $(CURDIR)/build
2424

2525
# When the length is no larger than this threshold, we consider the

‎test/.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
build
2+
download
3+
run

‎test/Makefile

+175
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# Build the `libc-test` tests as Wasm programs and run them with the selected
2+
# engine. Contributors beware! This Makefile follows the style of the
3+
# `libc-test` Makefile and uses some more exotic features of `make`.
4+
#
5+
# The top-level `test` target is composed of a chain of several phony
6+
# sub-targets:
7+
# - `download`: retrieve the `libc-test` source from a Git `$(MIRROR)`
8+
# - `build`: construct Wasm modules for a subset of the `libc-test` tests
9+
# - `run`: execute the benchmarks with a Wasm `$(ENGINE)` of choice (e.g.,
10+
# Wasmtime)
11+
12+
test: run
13+
14+
# Unlike the `libc-test` directory, we will output all artifacts to the `build`
15+
# directory (keeping with the `wasi-libc` conventions).
16+
OBJDIR ?= $(CURDIR)/build
17+
DOWNDIR ?= $(CURDIR)/download
18+
19+
##### DOWNLOAD #################################################################
20+
21+
LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
22+
LIBC_TEST = $(DOWNDIR)/libc-test
23+
LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-16/libclang_rt.builtins-wasm32-wasi-16.0.tar.gz
24+
LIBRT = $(DOWNDIR)/lib/wasi/libclang_rt.builtins-wasm32.a
25+
WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v3.0.0/wasmtime-v3.0.0-x86_64-linux.tar.xz
26+
WASMTIME = $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime
27+
28+
download: $(LIBC_TEST) $(LIBRT) $(WASMTIME)
29+
30+
$(DOWNDIR):
31+
mkdir -p download
32+
33+
$(LIBC_TEST): | $(DOWNDIR)
34+
git clone --depth 1 $(LIBC_TEST_URL) $@
35+
36+
# TODO install target to place into...
37+
$(LIBRT): | $(DOWNDIR)
38+
wget --no-clobber --directory-prefix=$(DOWNDIR) $(LIBRT_URL)
39+
tar --extract --file=$(DOWNDIR)/$(shell basename $(LIBRT_URL)) --directory=$(DOWNDIR)/
40+
41+
$(WASMTIME): | $(DOWNDIR)
42+
wget --no-clobber --directory-prefix=$(DOWNDIR) $(WASMTIME_URL)
43+
tar --extract --file=$(DOWNDIR)/$(shell basename $(WASMTIME_URL)) --directory=$(DOWNDIR)/
44+
45+
clean::
46+
rm -rf download
47+
48+
##### BUILD ####################################################################
49+
50+
# For now, we list out the tests that we can currently build and run. This is
51+
# heavily focused on the functional tests; in the future it would be good to
52+
# fill out the missing tests and also include some `src/api` and `src/math`
53+
# tests (TODO).
54+
TESTS := \
55+
$(LIBC_TEST)/src/functional/argv.c \
56+
$(LIBC_TEST)/src/functional/basename.c \
57+
$(LIBC_TEST)/src/functional/clocale_mbfuncs.c \
58+
$(LIBC_TEST)/src/functional/clock_gettime.c \
59+
$(LIBC_TEST)/src/functional/crypt.c \
60+
$(LIBC_TEST)/src/functional/dirname.c \
61+
$(LIBC_TEST)/src/functional/env.c \
62+
$(LIBC_TEST)/src/functional/fnmatch.c \
63+
$(LIBC_TEST)/src/functional/iconv_open.c \
64+
$(LIBC_TEST)/src/functional/mbc.c \
65+
$(LIBC_TEST)/src/functional/memstream.c \
66+
$(LIBC_TEST)/src/functional/qsort.c \
67+
$(LIBC_TEST)/src/functional/random.c \
68+
$(LIBC_TEST)/src/functional/search_hsearch.c \
69+
$(LIBC_TEST)/src/functional/search_insque.c \
70+
$(LIBC_TEST)/src/functional/search_lsearch.c \
71+
$(LIBC_TEST)/src/functional/search_tsearch.c \
72+
$(LIBC_TEST)/src/functional/snprintf.c \
73+
$(LIBC_TEST)/src/functional/sscanf.c \
74+
$(LIBC_TEST)/src/functional/strftime.c \
75+
$(LIBC_TEST)/src/functional/string.c \
76+
$(LIBC_TEST)/src/functional/string_memcpy.c \
77+
$(LIBC_TEST)/src/functional/string_memmem.c \
78+
$(LIBC_TEST)/src/functional/string_memset.c \
79+
$(LIBC_TEST)/src/functional/string_strchr.c \
80+
$(LIBC_TEST)/src/functional/string_strcspn.c \
81+
$(LIBC_TEST)/src/functional/string_strstr.c \
82+
$(LIBC_TEST)/src/functional/strtod.c \
83+
$(LIBC_TEST)/src/functional/strtod_long.c \
84+
$(LIBC_TEST)/src/functional/strtod_simple.c \
85+
$(LIBC_TEST)/src/functional/strtof.c \
86+
$(LIBC_TEST)/src/functional/strtol.c \
87+
$(LIBC_TEST)/src/functional/swprintf.c \
88+
$(LIBC_TEST)/src/functional/tgmath.c \
89+
$(LIBC_TEST)/src/functional/udiv.c \
90+
$(LIBC_TEST)/src/functional/wcsstr.c \
91+
$(LIBC_TEST)/src/functional/wcstol.c
92+
93+
# Part of the problem including more tests is that the `libc-test`
94+
# infrastructure code is not all Wasm-compilable. As we include more tests
95+
# above, this list will also likely need to grow.
96+
COMMON_TEST_INFRA = \
97+
$(LIBC_TEST)/src/common/path.c \
98+
$(LIBC_TEST)/src/common/print.c \
99+
$(LIBC_TEST)/src/common/rand.c \
100+
$(LIBC_TEST)/src/common/utf8.c
101+
102+
# Create various lists containing the various artifacts to be built: mainly,
103+
# $(WASM_OBJS) are compiled in the $(OBJDIRS) and then linked together to form
104+
# the $(WASMS) tests.
105+
NAMES := $(TESTS:$(LIBC_TEST)/src/%.c=%)
106+
WASMS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm)
107+
WASM_OBJS := $(TESTS:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
108+
INFRA_WASM_OBJS := $(COMMON_TEST_INFRA:$(LIBC_TEST)/src/%.c=$(OBJDIR)/%.wasm.o)
109+
WASM_OBJS += $(INFRA_WASM_OBJS)
110+
DIRS := $(patsubst $(OBJDIR)/%/,%,$(sort $(dir $(WASM_OBJS))))
111+
OBJDIRS := $(DIRS:%=$(OBJDIR)/%)
112+
113+
# Allow $(CC) to be set from the command line; ?= doesn't work for CC because
114+
# make has a default value for it.
115+
ifeq ($(origin CC), default)
116+
CC := clang
117+
endif
118+
LDFLAGS ?=
119+
CFLAGS ?= --target=wasm32-wasi --sysroot=../sysroot
120+
# Always include the `libc-test` infrastructure headers.
121+
CFLAGS += -I$(LIBC_TEST)/src/common
122+
123+
# Compile each selected test using Clang. Note that failures here are likely
124+
# due to a missing `libclang_rt.builtins-wasm32.a` in the Clang lib directory.
125+
# This location is system-dependent, but could be fixed by something like:
126+
# $ sudo mkdir /usr/lib64/clang/14.0.5/lib/wasi
127+
# $ sudo cp download/lib/wasi/libclang_rt.builtins-wasm32.a /usr/lib64/clang/14.0.5/lib/wasi/
128+
build: download $(WASMS)
129+
130+
$(WASMS): | $(OBJDIRS)
131+
$(OBJDIR)/%.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
132+
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^
133+
134+
$(WASM_OBJS): $(LIBC_TEST)/src/common/test.h | $(OBJDIRS)
135+
$(OBJDIR)/%.wasm.o: $(LIBC_TEST)/src/%.c
136+
$(CC) $(CFLAGS) -c -o $@ $<
137+
138+
$(OBJDIRS):
139+
mkdir -p $@
140+
141+
clean::
142+
rm -rf $(OBJDIR)
143+
144+
##### RUN ######################################################################
145+
146+
ENGINE ?= $(WASMTIME) run
147+
ERRS:=$(WASMS:%.wasm=%.wasm.err)
148+
149+
# Use the provided Wasm engine to execute each test, emitting its output into
150+
# a `.err` file.
151+
run: build $(ERRS)
152+
@echo "Tests passed"
153+
154+
$(ERRS): | $(OBJDIRS)
155+
156+
%.wasm.err: %.wasm
157+
$(ENGINE) $< >$@
158+
159+
clean::
160+
rm -rf $(OBJDIR)/*/*.err
161+
162+
##### MISC #####################################################################
163+
164+
# Note: the `clean` target has been built up by all of the previous sections.
165+
166+
debug:
167+
@echo NAMES $(NAMES)
168+
@echo TESTS $(TESTS)
169+
@echo WASMS $(WASMS)
170+
@echo WASM_OBJS $(WASM_OBJS)
171+
@echo ERRS $(ERRS)
172+
@echo DIRS $(DIRS)
173+
@echo OBJDIRS $(OBJDIRS)
174+
175+
.PHONY: test download build run clean

‎test/README.md

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Test
2+
3+
This directory runs a subset of libc-test using the sysroot produced by
4+
`wasi-libc`.
5+
6+
[libc-test]: https://wiki.musl-libc.org/libc-test.html

0 commit comments

Comments
 (0)
Please sign in to comment.