diff --git a/.github/actions/freebsd/action.yml b/.github/actions/freebsd/action.yml index 98163d28a82e9..79187175d739f 100644 --- a/.github/actions/freebsd/action.yml +++ b/.github/actions/freebsd/action.yml @@ -41,9 +41,11 @@ runs: webp \ libavif \ `#sqlite3` \ - curl + curl \ + gcc ./buildconf -f + CC=clang CXX=clang++ \ ./configure \ --prefix=/usr/local \ --enable-debug \ @@ -108,3 +110,6 @@ runs: --show-slow 1000 \ --set-timeout 120 \ -d zend_extension=opcache.so + + export TLSC=$(pwd)/ext/opcache/jit/tls/zend_jit_tls_x86_64.c + ./ext/opcache/jit/tls/testing/test.sh diff --git a/.github/actions/test-tls-alpine/action.yml b/.github/actions/test-tls-alpine/action.yml new file mode 100644 index 0000000000000..3718aae6f50dc --- /dev/null +++ b/.github/actions/test-tls-alpine/action.yml @@ -0,0 +1,10 @@ +name: Test TLS +runs: + using: composite + steps: + - shell: bash + run: | + set -x + apk add clang gcc binutils-gold lld + export TLSC=$(pwd)/ext/opcache/jit/tls/zend_jit_tls_x86_64.c + ./ext/opcache/jit/tls/testing/test.sh diff --git a/.github/actions/test-tls-macos/action.yml b/.github/actions/test-tls-macos/action.yml new file mode 100644 index 0000000000000..9cf767c687609 --- /dev/null +++ b/.github/actions/test-tls-macos/action.yml @@ -0,0 +1,9 @@ +name: Test TLS +runs: + using: composite + steps: + - shell: bash + run: | + set -x + export TLSC=$(pwd)/ext/opcache/jit/tls/zend_jit_tls_darwin.c + ./ext/opcache/jit/tls/testing/test.sh diff --git a/.github/actions/test-tls-x32/action.yml b/.github/actions/test-tls-x32/action.yml new file mode 100644 index 0000000000000..ba3e72dd915ec --- /dev/null +++ b/.github/actions/test-tls-x32/action.yml @@ -0,0 +1,12 @@ +name: Test TLS +runs: + using: composite + steps: + - shell: bash + run: | + set -x + export DEBIAN_FRONTEND=noninteractive + apt-get install -y gcc clang lld + export TLSC=$(pwd)/ext/opcache/jit/tls/zend_jit_tls_x86.c + export MACHINE=-m32 + ./ext/opcache/jit/tls/testing/test.sh diff --git a/.github/actions/test-tls-x64/action.yml b/.github/actions/test-tls-x64/action.yml new file mode 100644 index 0000000000000..2ca49501c662f --- /dev/null +++ b/.github/actions/test-tls-x64/action.yml @@ -0,0 +1,11 @@ +name: Test TLS +runs: + using: composite + steps: + - shell: bash + run: | + set -x + export DEBIAN_FRONTEND=noninteractive + sudo apt-get install -y gcc clang lld + export TLSC=$(pwd)/ext/opcache/jit/tls/zend_jit_tls_x86_64.c + ./ext/opcache/jit/tls/testing/test.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1167b9c5d2934..97b25a8d5ccf3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -131,6 +131,8 @@ jobs: --asan -x -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Test TLS resolution + uses: ./.github/actions/test-tls-alpine - name: Notify Slack if: failure() uses: ./.github/actions/notify-slack @@ -263,6 +265,9 @@ jobs: ${{ matrix.run_tests_parameters }} -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Test TLS resolution + if: matrix.debug && matrix.zts && !matrix.asan + uses: ./.github/actions/test-tls-x64 - name: Verify generated files are up to date uses: ./.github/actions/verify-generated-files - name: Notify Slack @@ -352,6 +357,9 @@ jobs: ${{ matrix.run_tests_parameters }} -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Test TLS resolution + if: matrix.debug && matrix.zts + uses: ./.github/actions/test-tls-x32 - name: Notify Slack if: failure() uses: ./.github/actions/notify-slack @@ -411,6 +419,10 @@ jobs: runTestsParameters: >- -d zend_extension=opcache.so -d opcache.enable_cli=1 + - name: Test TLS resolution + # JIT+ZTS not supported yet on ARM64 + if: matrix.os != '14' && matrix.debug && matrix.zts + uses: ./.github/actions/test-tls-macos - name: Verify generated files are up to date uses: ./.github/actions/verify-generated-files - name: Notify Slack diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 8f6d5ab711b28..a7b35351c6ed6 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -73,19 +73,23 @@ if test "$PHP_OPCACHE" != "no"; then IR_TARGET=IR_TARGET_X64 DASM_FLAGS="-D X64APPLE=1 -D X64=1" DASM_ARCH="x86" + TLS_TARGET="darwin" ], [*x86_64*|amd64-*-freebsd*], [ IR_TARGET=IR_TARGET_X64 DASM_FLAGS="-D X64=1" DASM_ARCH="x86" + TLS_TARGET="x86_64" ], [[i[34567]86*|x86*]], [ IR_TARGET=IR_TARGET_X86 DASM_ARCH="x86" + TLS_TARGET="x86" ], [aarch64*], [ IR_TARGET=IR_TARGET_AARCH64 DASM_ARCH="aarch64" + TLS_TARGET="aarch64" ]) AS_VAR_IF([PHP_CAPSTONE], [yes], @@ -102,6 +106,10 @@ if test "$PHP_OPCACHE" != "no"; then JIT_CFLAGS="-I@ext_builddir@/jit/ir -D$IR_TARGET -DIR_PHP" AS_VAR_IF([ZEND_DEBUG], [yes], [JIT_CFLAGS="$JIT_CFLAGS -DIR_DEBUG"]) + + AS_VAR_IF([PHP_THREAD_SAFETY], [yes], [ + ZEND_JIT_SRC="$ZEND_JIT_SRC jit/tls/zend_jit_tls_$TLS_TARGET.c" + ]) ]) AC_CHECK_FUNCS([mprotect shm_create_largepage]) diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index fa89ca1f18a39..2ae2c66474c3e 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -33,6 +33,7 @@ if (PHP_OPCACHE != "no") { DEFINE("IR_TARGET", ir_target); DEFINE("DASM_FLAGS", dasm_flags); DEFINE("DASM_ARCH", "x86"); + DEFINE("TLS_TARGET", "win"); AC_DEFINE('HAVE_JIT', 1, 'Define to 1 to enable JIT.'); @@ -52,6 +53,11 @@ if (PHP_OPCACHE != "no") { ADD_SOURCES(configure_module_dirname + "\\jit", "zend_jit.c zend_jit_vm_helpers.c", "opcache", "ext\\opcache\\jit"); + if (PHP_ZTS == "yes") { + ADD_SOURCES(configure_module_dirname + "\\jit\\tls", + "zend_jit_tls_win.c", + "opcache", "ext\\opcache\\jit\\tls"); + } ADD_SOURCES(configure_module_dirname + "\\jit\\ir", "ir.c", "opcache", "ext\\opcache\\jit\\ir"); ADD_SOURCES(configure_module_dirname + "\\jit\\ir", diff --git a/ext/opcache/jit/tls/testing/.gitignore b/ext/opcache/jit/tls/testing/.gitignore new file mode 100644 index 0000000000000..3873646140f91 --- /dev/null +++ b/ext/opcache/jit/tls/testing/.gitignore @@ -0,0 +1,4 @@ +*.so +*.o +main +main.dSYM diff --git a/ext/opcache/jit/tls/testing/def-vars.h b/ext/opcache/jit/tls/testing/def-vars.h new file mode 100644 index 0000000000000..66cdc2442b8cd --- /dev/null +++ b/ext/opcache/jit/tls/testing/def-vars.h @@ -0,0 +1,1030 @@ +/* Declare a few additional TLS variables to fill any surplus space, + * so _tsrm_ls_cache is allocated in the dynamic section. */ + +#define DEF_VAR(prefix, num) __thread void* prefix##num +#define DEF_VARS(prefix) \ + DEF_VAR(prefix, 0000); \ + DEF_VAR(prefix, 0001); \ + DEF_VAR(prefix, 0002); \ + DEF_VAR(prefix, 0003); \ + DEF_VAR(prefix, 0004); \ + DEF_VAR(prefix, 0005); \ + DEF_VAR(prefix, 0006); \ + DEF_VAR(prefix, 0007); \ + DEF_VAR(prefix, 0008); \ + DEF_VAR(prefix, 0009); \ + DEF_VAR(prefix, 0010); \ + DEF_VAR(prefix, 0011); \ + DEF_VAR(prefix, 0012); \ + DEF_VAR(prefix, 0013); \ + DEF_VAR(prefix, 0014); \ + DEF_VAR(prefix, 0015); \ + DEF_VAR(prefix, 0016); \ + DEF_VAR(prefix, 0017); \ + DEF_VAR(prefix, 0018); \ + DEF_VAR(prefix, 0019); \ + DEF_VAR(prefix, 0020); \ + DEF_VAR(prefix, 0021); \ + DEF_VAR(prefix, 0022); \ + DEF_VAR(prefix, 0023); \ + DEF_VAR(prefix, 0024); \ + DEF_VAR(prefix, 0025); \ + DEF_VAR(prefix, 0026); \ + DEF_VAR(prefix, 0027); \ + DEF_VAR(prefix, 0028); \ + DEF_VAR(prefix, 0029); \ + DEF_VAR(prefix, 0030); \ + DEF_VAR(prefix, 0031); \ + DEF_VAR(prefix, 0032); \ + DEF_VAR(prefix, 0033); \ + DEF_VAR(prefix, 0034); \ + DEF_VAR(prefix, 0035); \ + DEF_VAR(prefix, 0036); \ + DEF_VAR(prefix, 0037); \ + DEF_VAR(prefix, 0038); \ + DEF_VAR(prefix, 0039); \ + DEF_VAR(prefix, 0040); \ + DEF_VAR(prefix, 0041); \ + DEF_VAR(prefix, 0042); \ + DEF_VAR(prefix, 0043); \ + DEF_VAR(prefix, 0044); \ + DEF_VAR(prefix, 0045); \ + DEF_VAR(prefix, 0046); \ + DEF_VAR(prefix, 0047); \ + DEF_VAR(prefix, 0048); \ + DEF_VAR(prefix, 0049); \ + DEF_VAR(prefix, 0050); \ + DEF_VAR(prefix, 0051); \ + DEF_VAR(prefix, 0052); \ + DEF_VAR(prefix, 0053); \ + DEF_VAR(prefix, 0054); \ + DEF_VAR(prefix, 0055); \ + DEF_VAR(prefix, 0056); \ + DEF_VAR(prefix, 0057); \ + DEF_VAR(prefix, 0058); \ + DEF_VAR(prefix, 0059); \ + DEF_VAR(prefix, 0060); \ + DEF_VAR(prefix, 0061); \ + DEF_VAR(prefix, 0062); \ + DEF_VAR(prefix, 0063); \ + DEF_VAR(prefix, 0064); \ + DEF_VAR(prefix, 0065); \ + DEF_VAR(prefix, 0066); \ + DEF_VAR(prefix, 0067); \ + DEF_VAR(prefix, 0068); \ + DEF_VAR(prefix, 0069); \ + DEF_VAR(prefix, 0070); \ + DEF_VAR(prefix, 0071); \ + DEF_VAR(prefix, 0072); \ + DEF_VAR(prefix, 0073); \ + DEF_VAR(prefix, 0074); \ + DEF_VAR(prefix, 0075); \ + DEF_VAR(prefix, 0076); \ + DEF_VAR(prefix, 0077); \ + DEF_VAR(prefix, 0078); \ + DEF_VAR(prefix, 0079); \ + DEF_VAR(prefix, 0080); \ + DEF_VAR(prefix, 0081); \ + DEF_VAR(prefix, 0082); \ + DEF_VAR(prefix, 0083); \ + DEF_VAR(prefix, 0084); \ + DEF_VAR(prefix, 0085); \ + DEF_VAR(prefix, 0086); \ + DEF_VAR(prefix, 0087); \ + DEF_VAR(prefix, 0088); \ + DEF_VAR(prefix, 0089); \ + DEF_VAR(prefix, 0090); \ + DEF_VAR(prefix, 0091); \ + DEF_VAR(prefix, 0092); \ + DEF_VAR(prefix, 0093); \ + DEF_VAR(prefix, 0094); \ + DEF_VAR(prefix, 0095); \ + DEF_VAR(prefix, 0096); \ + DEF_VAR(prefix, 0097); \ + DEF_VAR(prefix, 0098); \ + DEF_VAR(prefix, 0099); \ + DEF_VAR(prefix, 0100); \ + DEF_VAR(prefix, 0101); \ + DEF_VAR(prefix, 0102); \ + DEF_VAR(prefix, 0103); \ + DEF_VAR(prefix, 0104); \ + DEF_VAR(prefix, 0105); \ + DEF_VAR(prefix, 0106); \ + DEF_VAR(prefix, 0107); \ + DEF_VAR(prefix, 0108); \ + DEF_VAR(prefix, 0109); \ + DEF_VAR(prefix, 0110); \ + DEF_VAR(prefix, 0111); \ + DEF_VAR(prefix, 0112); \ + DEF_VAR(prefix, 0113); \ + DEF_VAR(prefix, 0114); \ + DEF_VAR(prefix, 0115); \ + DEF_VAR(prefix, 0116); \ + DEF_VAR(prefix, 0117); \ + DEF_VAR(prefix, 0118); \ + DEF_VAR(prefix, 0119); \ + DEF_VAR(prefix, 0120); \ + DEF_VAR(prefix, 0121); \ + DEF_VAR(prefix, 0122); \ + DEF_VAR(prefix, 0123); \ + DEF_VAR(prefix, 0124); \ + DEF_VAR(prefix, 0125); \ + DEF_VAR(prefix, 0126); \ + DEF_VAR(prefix, 0127); \ + DEF_VAR(prefix, 0128); \ + DEF_VAR(prefix, 0129); \ + DEF_VAR(prefix, 0130); \ + DEF_VAR(prefix, 0131); \ + DEF_VAR(prefix, 0132); \ + DEF_VAR(prefix, 0133); \ + DEF_VAR(prefix, 0134); \ + DEF_VAR(prefix, 0135); \ + DEF_VAR(prefix, 0136); \ + DEF_VAR(prefix, 0137); \ + DEF_VAR(prefix, 0138); \ + DEF_VAR(prefix, 0139); \ + DEF_VAR(prefix, 0140); \ + DEF_VAR(prefix, 0141); \ + DEF_VAR(prefix, 0142); \ + DEF_VAR(prefix, 0143); \ + DEF_VAR(prefix, 0144); \ + DEF_VAR(prefix, 0145); \ + DEF_VAR(prefix, 0146); \ + DEF_VAR(prefix, 0147); \ + DEF_VAR(prefix, 0148); \ + DEF_VAR(prefix, 0149); \ + DEF_VAR(prefix, 0150); \ + DEF_VAR(prefix, 0151); \ + DEF_VAR(prefix, 0152); \ + DEF_VAR(prefix, 0153); \ + DEF_VAR(prefix, 0154); \ + DEF_VAR(prefix, 0155); \ + DEF_VAR(prefix, 0156); \ + DEF_VAR(prefix, 0157); \ + DEF_VAR(prefix, 0158); \ + DEF_VAR(prefix, 0159); \ + DEF_VAR(prefix, 0160); \ + DEF_VAR(prefix, 0161); \ + DEF_VAR(prefix, 0162); \ + DEF_VAR(prefix, 0163); \ + DEF_VAR(prefix, 0164); \ + DEF_VAR(prefix, 0165); \ + DEF_VAR(prefix, 0166); \ + DEF_VAR(prefix, 0167); \ + DEF_VAR(prefix, 0168); \ + DEF_VAR(prefix, 0169); \ + DEF_VAR(prefix, 0170); \ + DEF_VAR(prefix, 0171); \ + DEF_VAR(prefix, 0172); \ + DEF_VAR(prefix, 0173); \ + DEF_VAR(prefix, 0174); \ + DEF_VAR(prefix, 0175); \ + DEF_VAR(prefix, 0176); \ + DEF_VAR(prefix, 0177); \ + DEF_VAR(prefix, 0178); \ + DEF_VAR(prefix, 0179); \ + DEF_VAR(prefix, 0180); \ + DEF_VAR(prefix, 0181); \ + DEF_VAR(prefix, 0182); \ + DEF_VAR(prefix, 0183); \ + DEF_VAR(prefix, 0184); \ + DEF_VAR(prefix, 0185); \ + DEF_VAR(prefix, 0186); \ + DEF_VAR(prefix, 0187); \ + DEF_VAR(prefix, 0188); \ + DEF_VAR(prefix, 0189); \ + DEF_VAR(prefix, 0190); \ + DEF_VAR(prefix, 0191); \ + DEF_VAR(prefix, 0192); \ + DEF_VAR(prefix, 0193); \ + DEF_VAR(prefix, 0194); \ + DEF_VAR(prefix, 0195); \ + DEF_VAR(prefix, 0196); \ + DEF_VAR(prefix, 0197); \ + DEF_VAR(prefix, 0198); \ + DEF_VAR(prefix, 0199); \ + DEF_VAR(prefix, 0200); \ + DEF_VAR(prefix, 0201); \ + DEF_VAR(prefix, 0202); \ + DEF_VAR(prefix, 0203); \ + DEF_VAR(prefix, 0204); \ + DEF_VAR(prefix, 0205); \ + DEF_VAR(prefix, 0206); \ + DEF_VAR(prefix, 0207); \ + DEF_VAR(prefix, 0208); \ + DEF_VAR(prefix, 0209); \ + DEF_VAR(prefix, 0210); \ + DEF_VAR(prefix, 0211); \ + DEF_VAR(prefix, 0212); \ + DEF_VAR(prefix, 0213); \ + DEF_VAR(prefix, 0214); \ + DEF_VAR(prefix, 0215); \ + DEF_VAR(prefix, 0216); \ + DEF_VAR(prefix, 0217); \ + DEF_VAR(prefix, 0218); \ + DEF_VAR(prefix, 0219); \ + DEF_VAR(prefix, 0220); \ + DEF_VAR(prefix, 0221); \ + DEF_VAR(prefix, 0222); \ + DEF_VAR(prefix, 0223); \ + DEF_VAR(prefix, 0224); \ + DEF_VAR(prefix, 0225); \ + DEF_VAR(prefix, 0226); \ + DEF_VAR(prefix, 0227); \ + DEF_VAR(prefix, 0228); \ + DEF_VAR(prefix, 0229); \ + DEF_VAR(prefix, 0230); \ + DEF_VAR(prefix, 0231); \ + DEF_VAR(prefix, 0232); \ + DEF_VAR(prefix, 0233); \ + DEF_VAR(prefix, 0234); \ + DEF_VAR(prefix, 0235); \ + DEF_VAR(prefix, 0236); \ + DEF_VAR(prefix, 0237); \ + DEF_VAR(prefix, 0238); \ + DEF_VAR(prefix, 0239); \ + DEF_VAR(prefix, 0240); \ + DEF_VAR(prefix, 0241); \ + DEF_VAR(prefix, 0242); \ + DEF_VAR(prefix, 0243); \ + DEF_VAR(prefix, 0244); \ + DEF_VAR(prefix, 0245); \ + DEF_VAR(prefix, 0246); \ + DEF_VAR(prefix, 0247); \ + DEF_VAR(prefix, 0248); \ + DEF_VAR(prefix, 0249); \ + DEF_VAR(prefix, 0250); \ + DEF_VAR(prefix, 0251); \ + DEF_VAR(prefix, 0252); \ + DEF_VAR(prefix, 0253); \ + DEF_VAR(prefix, 0254); \ + DEF_VAR(prefix, 0255); \ + DEF_VAR(prefix, 0256); \ + DEF_VAR(prefix, 0257); \ + DEF_VAR(prefix, 0258); \ + DEF_VAR(prefix, 0259); \ + DEF_VAR(prefix, 0260); \ + DEF_VAR(prefix, 0261); \ + DEF_VAR(prefix, 0262); \ + DEF_VAR(prefix, 0263); \ + DEF_VAR(prefix, 0264); \ + DEF_VAR(prefix, 0265); \ + DEF_VAR(prefix, 0266); \ + DEF_VAR(prefix, 0267); \ + DEF_VAR(prefix, 0268); \ + DEF_VAR(prefix, 0269); \ + DEF_VAR(prefix, 0270); \ + DEF_VAR(prefix, 0271); \ + DEF_VAR(prefix, 0272); \ + DEF_VAR(prefix, 0273); \ + DEF_VAR(prefix, 0274); \ + DEF_VAR(prefix, 0275); \ + DEF_VAR(prefix, 0276); \ + DEF_VAR(prefix, 0277); \ + DEF_VAR(prefix, 0278); \ + DEF_VAR(prefix, 0279); \ + DEF_VAR(prefix, 0280); \ + DEF_VAR(prefix, 0281); \ + DEF_VAR(prefix, 0282); \ + DEF_VAR(prefix, 0283); \ + DEF_VAR(prefix, 0284); \ + DEF_VAR(prefix, 0285); \ + DEF_VAR(prefix, 0286); \ + DEF_VAR(prefix, 0287); \ + DEF_VAR(prefix, 0288); \ + DEF_VAR(prefix, 0289); \ + DEF_VAR(prefix, 0290); \ + DEF_VAR(prefix, 0291); \ + DEF_VAR(prefix, 0292); \ + DEF_VAR(prefix, 0293); \ + DEF_VAR(prefix, 0294); \ + DEF_VAR(prefix, 0295); \ + DEF_VAR(prefix, 0296); \ + DEF_VAR(prefix, 0297); \ + DEF_VAR(prefix, 0298); \ + DEF_VAR(prefix, 0299); \ + DEF_VAR(prefix, 0300); \ + DEF_VAR(prefix, 0301); \ + DEF_VAR(prefix, 0302); \ + DEF_VAR(prefix, 0303); \ + DEF_VAR(prefix, 0304); \ + DEF_VAR(prefix, 0305); \ + DEF_VAR(prefix, 0306); \ + DEF_VAR(prefix, 0307); \ + DEF_VAR(prefix, 0308); \ + DEF_VAR(prefix, 0309); \ + DEF_VAR(prefix, 0310); \ + DEF_VAR(prefix, 0311); \ + DEF_VAR(prefix, 0312); \ + DEF_VAR(prefix, 0313); \ + DEF_VAR(prefix, 0314); \ + DEF_VAR(prefix, 0315); \ + DEF_VAR(prefix, 0316); \ + DEF_VAR(prefix, 0317); \ + DEF_VAR(prefix, 0318); \ + DEF_VAR(prefix, 0319); \ + DEF_VAR(prefix, 0320); \ + DEF_VAR(prefix, 0321); \ + DEF_VAR(prefix, 0322); \ + DEF_VAR(prefix, 0323); \ + DEF_VAR(prefix, 0324); \ + DEF_VAR(prefix, 0325); \ + DEF_VAR(prefix, 0326); \ + DEF_VAR(prefix, 0327); \ + DEF_VAR(prefix, 0328); \ + DEF_VAR(prefix, 0329); \ + DEF_VAR(prefix, 0330); \ + DEF_VAR(prefix, 0331); \ + DEF_VAR(prefix, 0332); \ + DEF_VAR(prefix, 0333); \ + DEF_VAR(prefix, 0334); \ + DEF_VAR(prefix, 0335); \ + DEF_VAR(prefix, 0336); \ + DEF_VAR(prefix, 0337); \ + DEF_VAR(prefix, 0338); \ + DEF_VAR(prefix, 0339); \ + DEF_VAR(prefix, 0340); \ + DEF_VAR(prefix, 0341); \ + DEF_VAR(prefix, 0342); \ + DEF_VAR(prefix, 0343); \ + DEF_VAR(prefix, 0344); \ + DEF_VAR(prefix, 0345); \ + DEF_VAR(prefix, 0346); \ + DEF_VAR(prefix, 0347); \ + DEF_VAR(prefix, 0348); \ + DEF_VAR(prefix, 0349); \ + DEF_VAR(prefix, 0350); \ + DEF_VAR(prefix, 0351); \ + DEF_VAR(prefix, 0352); \ + DEF_VAR(prefix, 0353); \ + DEF_VAR(prefix, 0354); \ + DEF_VAR(prefix, 0355); \ + DEF_VAR(prefix, 0356); \ + DEF_VAR(prefix, 0357); \ + DEF_VAR(prefix, 0358); \ + DEF_VAR(prefix, 0359); \ + DEF_VAR(prefix, 0360); \ + DEF_VAR(prefix, 0361); \ + DEF_VAR(prefix, 0362); \ + DEF_VAR(prefix, 0363); \ + DEF_VAR(prefix, 0364); \ + DEF_VAR(prefix, 0365); \ + DEF_VAR(prefix, 0366); \ + DEF_VAR(prefix, 0367); \ + DEF_VAR(prefix, 0368); \ + DEF_VAR(prefix, 0369); \ + DEF_VAR(prefix, 0370); \ + DEF_VAR(prefix, 0371); \ + DEF_VAR(prefix, 0372); \ + DEF_VAR(prefix, 0373); \ + DEF_VAR(prefix, 0374); \ + DEF_VAR(prefix, 0375); \ + DEF_VAR(prefix, 0376); \ + DEF_VAR(prefix, 0377); \ + DEF_VAR(prefix, 0378); \ + DEF_VAR(prefix, 0379); \ + DEF_VAR(prefix, 0380); \ + DEF_VAR(prefix, 0381); \ + DEF_VAR(prefix, 0382); \ + DEF_VAR(prefix, 0383); \ + DEF_VAR(prefix, 0384); \ + DEF_VAR(prefix, 0385); \ + DEF_VAR(prefix, 0386); \ + DEF_VAR(prefix, 0387); \ + DEF_VAR(prefix, 0388); \ + DEF_VAR(prefix, 0389); \ + DEF_VAR(prefix, 0390); \ + DEF_VAR(prefix, 0391); \ + DEF_VAR(prefix, 0392); \ + DEF_VAR(prefix, 0393); \ + DEF_VAR(prefix, 0394); \ + DEF_VAR(prefix, 0395); \ + DEF_VAR(prefix, 0396); \ + DEF_VAR(prefix, 0397); \ + DEF_VAR(prefix, 0398); \ + DEF_VAR(prefix, 0399); \ + DEF_VAR(prefix, 0400); \ + DEF_VAR(prefix, 0401); \ + DEF_VAR(prefix, 0402); \ + DEF_VAR(prefix, 0403); \ + DEF_VAR(prefix, 0404); \ + DEF_VAR(prefix, 0405); \ + DEF_VAR(prefix, 0406); \ + DEF_VAR(prefix, 0407); \ + DEF_VAR(prefix, 0408); \ + DEF_VAR(prefix, 0409); \ + DEF_VAR(prefix, 0410); \ + DEF_VAR(prefix, 0411); \ + DEF_VAR(prefix, 0412); \ + DEF_VAR(prefix, 0413); \ + DEF_VAR(prefix, 0414); \ + DEF_VAR(prefix, 0415); \ + DEF_VAR(prefix, 0416); \ + DEF_VAR(prefix, 0417); \ + DEF_VAR(prefix, 0418); \ + DEF_VAR(prefix, 0419); \ + DEF_VAR(prefix, 0420); \ + DEF_VAR(prefix, 0421); \ + DEF_VAR(prefix, 0422); \ + DEF_VAR(prefix, 0423); \ + DEF_VAR(prefix, 0424); \ + DEF_VAR(prefix, 0425); \ + DEF_VAR(prefix, 0426); \ + DEF_VAR(prefix, 0427); \ + DEF_VAR(prefix, 0428); \ + DEF_VAR(prefix, 0429); \ + DEF_VAR(prefix, 0430); \ + DEF_VAR(prefix, 0431); \ + DEF_VAR(prefix, 0432); \ + DEF_VAR(prefix, 0433); \ + DEF_VAR(prefix, 0434); \ + DEF_VAR(prefix, 0435); \ + DEF_VAR(prefix, 0436); \ + DEF_VAR(prefix, 0437); \ + DEF_VAR(prefix, 0438); \ + DEF_VAR(prefix, 0439); \ + DEF_VAR(prefix, 0440); \ + DEF_VAR(prefix, 0441); \ + DEF_VAR(prefix, 0442); \ + DEF_VAR(prefix, 0443); \ + DEF_VAR(prefix, 0444); \ + DEF_VAR(prefix, 0445); \ + DEF_VAR(prefix, 0446); \ + DEF_VAR(prefix, 0447); \ + DEF_VAR(prefix, 0448); \ + DEF_VAR(prefix, 0449); \ + DEF_VAR(prefix, 0450); \ + DEF_VAR(prefix, 0451); \ + DEF_VAR(prefix, 0452); \ + DEF_VAR(prefix, 0453); \ + DEF_VAR(prefix, 0454); \ + DEF_VAR(prefix, 0455); \ + DEF_VAR(prefix, 0456); \ + DEF_VAR(prefix, 0457); \ + DEF_VAR(prefix, 0458); \ + DEF_VAR(prefix, 0459); \ + DEF_VAR(prefix, 0460); \ + DEF_VAR(prefix, 0461); \ + DEF_VAR(prefix, 0462); \ + DEF_VAR(prefix, 0463); \ + DEF_VAR(prefix, 0464); \ + DEF_VAR(prefix, 0465); \ + DEF_VAR(prefix, 0466); \ + DEF_VAR(prefix, 0467); \ + DEF_VAR(prefix, 0468); \ + DEF_VAR(prefix, 0469); \ + DEF_VAR(prefix, 0470); \ + DEF_VAR(prefix, 0471); \ + DEF_VAR(prefix, 0472); \ + DEF_VAR(prefix, 0473); \ + DEF_VAR(prefix, 0474); \ + DEF_VAR(prefix, 0475); \ + DEF_VAR(prefix, 0476); \ + DEF_VAR(prefix, 0477); \ + DEF_VAR(prefix, 0478); \ + DEF_VAR(prefix, 0479); \ + DEF_VAR(prefix, 0480); \ + DEF_VAR(prefix, 0481); \ + DEF_VAR(prefix, 0482); \ + DEF_VAR(prefix, 0483); \ + DEF_VAR(prefix, 0484); \ + DEF_VAR(prefix, 0485); \ + DEF_VAR(prefix, 0486); \ + DEF_VAR(prefix, 0487); \ + DEF_VAR(prefix, 0488); \ + DEF_VAR(prefix, 0489); \ + DEF_VAR(prefix, 0490); \ + DEF_VAR(prefix, 0491); \ + DEF_VAR(prefix, 0492); \ + DEF_VAR(prefix, 0493); \ + DEF_VAR(prefix, 0494); \ + DEF_VAR(prefix, 0495); \ + DEF_VAR(prefix, 0496); \ + DEF_VAR(prefix, 0497); \ + DEF_VAR(prefix, 0498); \ + DEF_VAR(prefix, 0499); \ + DEF_VAR(prefix, 0500); \ + DEF_VAR(prefix, 0501); \ + DEF_VAR(prefix, 0502); \ + DEF_VAR(prefix, 0503); \ + DEF_VAR(prefix, 0504); \ + DEF_VAR(prefix, 0505); \ + DEF_VAR(prefix, 0506); \ + DEF_VAR(prefix, 0507); \ + DEF_VAR(prefix, 0508); \ + DEF_VAR(prefix, 0509); \ + DEF_VAR(prefix, 0510); \ + DEF_VAR(prefix, 0511); \ + DEF_VAR(prefix, 0512); \ + DEF_VAR(prefix, 0513); \ + DEF_VAR(prefix, 0514); \ + DEF_VAR(prefix, 0515); \ + DEF_VAR(prefix, 0516); \ + DEF_VAR(prefix, 0517); \ + DEF_VAR(prefix, 0518); \ + DEF_VAR(prefix, 0519); \ + DEF_VAR(prefix, 0520); \ + DEF_VAR(prefix, 0521); \ + DEF_VAR(prefix, 0522); \ + DEF_VAR(prefix, 0523); \ + DEF_VAR(prefix, 0524); \ + DEF_VAR(prefix, 0525); \ + DEF_VAR(prefix, 0526); \ + DEF_VAR(prefix, 0527); \ + DEF_VAR(prefix, 0528); \ + DEF_VAR(prefix, 0529); \ + DEF_VAR(prefix, 0530); \ + DEF_VAR(prefix, 0531); \ + DEF_VAR(prefix, 0532); \ + DEF_VAR(prefix, 0533); \ + DEF_VAR(prefix, 0534); \ + DEF_VAR(prefix, 0535); \ + DEF_VAR(prefix, 0536); \ + DEF_VAR(prefix, 0537); \ + DEF_VAR(prefix, 0538); \ + DEF_VAR(prefix, 0539); \ + DEF_VAR(prefix, 0540); \ + DEF_VAR(prefix, 0541); \ + DEF_VAR(prefix, 0542); \ + DEF_VAR(prefix, 0543); \ + DEF_VAR(prefix, 0544); \ + DEF_VAR(prefix, 0545); \ + DEF_VAR(prefix, 0546); \ + DEF_VAR(prefix, 0547); \ + DEF_VAR(prefix, 0548); \ + DEF_VAR(prefix, 0549); \ + DEF_VAR(prefix, 0550); \ + DEF_VAR(prefix, 0551); \ + DEF_VAR(prefix, 0552); \ + DEF_VAR(prefix, 0553); \ + DEF_VAR(prefix, 0554); \ + DEF_VAR(prefix, 0555); \ + DEF_VAR(prefix, 0556); \ + DEF_VAR(prefix, 0557); \ + DEF_VAR(prefix, 0558); \ + DEF_VAR(prefix, 0559); \ + DEF_VAR(prefix, 0560); \ + DEF_VAR(prefix, 0561); \ + DEF_VAR(prefix, 0562); \ + DEF_VAR(prefix, 0563); \ + DEF_VAR(prefix, 0564); \ + DEF_VAR(prefix, 0565); \ + DEF_VAR(prefix, 0566); \ + DEF_VAR(prefix, 0567); \ + DEF_VAR(prefix, 0568); \ + DEF_VAR(prefix, 0569); \ + DEF_VAR(prefix, 0570); \ + DEF_VAR(prefix, 0571); \ + DEF_VAR(prefix, 0572); \ + DEF_VAR(prefix, 0573); \ + DEF_VAR(prefix, 0574); \ + DEF_VAR(prefix, 0575); \ + DEF_VAR(prefix, 0576); \ + DEF_VAR(prefix, 0577); \ + DEF_VAR(prefix, 0578); \ + DEF_VAR(prefix, 0579); \ + DEF_VAR(prefix, 0580); \ + DEF_VAR(prefix, 0581); \ + DEF_VAR(prefix, 0582); \ + DEF_VAR(prefix, 0583); \ + DEF_VAR(prefix, 0584); \ + DEF_VAR(prefix, 0585); \ + DEF_VAR(prefix, 0586); \ + DEF_VAR(prefix, 0587); \ + DEF_VAR(prefix, 0588); \ + DEF_VAR(prefix, 0589); \ + DEF_VAR(prefix, 0590); \ + DEF_VAR(prefix, 0591); \ + DEF_VAR(prefix, 0592); \ + DEF_VAR(prefix, 0593); \ + DEF_VAR(prefix, 0594); \ + DEF_VAR(prefix, 0595); \ + DEF_VAR(prefix, 0596); \ + DEF_VAR(prefix, 0597); \ + DEF_VAR(prefix, 0598); \ + DEF_VAR(prefix, 0599); \ + DEF_VAR(prefix, 0600); \ + DEF_VAR(prefix, 0601); \ + DEF_VAR(prefix, 0602); \ + DEF_VAR(prefix, 0603); \ + DEF_VAR(prefix, 0604); \ + DEF_VAR(prefix, 0605); \ + DEF_VAR(prefix, 0606); \ + DEF_VAR(prefix, 0607); \ + DEF_VAR(prefix, 0608); \ + DEF_VAR(prefix, 0609); \ + DEF_VAR(prefix, 0610); \ + DEF_VAR(prefix, 0611); \ + DEF_VAR(prefix, 0612); \ + DEF_VAR(prefix, 0613); \ + DEF_VAR(prefix, 0614); \ + DEF_VAR(prefix, 0615); \ + DEF_VAR(prefix, 0616); \ + DEF_VAR(prefix, 0617); \ + DEF_VAR(prefix, 0618); \ + DEF_VAR(prefix, 0619); \ + DEF_VAR(prefix, 0620); \ + DEF_VAR(prefix, 0621); \ + DEF_VAR(prefix, 0622); \ + DEF_VAR(prefix, 0623); \ + DEF_VAR(prefix, 0624); \ + DEF_VAR(prefix, 0625); \ + DEF_VAR(prefix, 0626); \ + DEF_VAR(prefix, 0627); \ + DEF_VAR(prefix, 0628); \ + DEF_VAR(prefix, 0629); \ + DEF_VAR(prefix, 0630); \ + DEF_VAR(prefix, 0631); \ + DEF_VAR(prefix, 0632); \ + DEF_VAR(prefix, 0633); \ + DEF_VAR(prefix, 0634); \ + DEF_VAR(prefix, 0635); \ + DEF_VAR(prefix, 0636); \ + DEF_VAR(prefix, 0637); \ + DEF_VAR(prefix, 0638); \ + DEF_VAR(prefix, 0639); \ + DEF_VAR(prefix, 0640); \ + DEF_VAR(prefix, 0641); \ + DEF_VAR(prefix, 0642); \ + DEF_VAR(prefix, 0643); \ + DEF_VAR(prefix, 0644); \ + DEF_VAR(prefix, 0645); \ + DEF_VAR(prefix, 0646); \ + DEF_VAR(prefix, 0647); \ + DEF_VAR(prefix, 0648); \ + DEF_VAR(prefix, 0649); \ + DEF_VAR(prefix, 0650); \ + DEF_VAR(prefix, 0651); \ + DEF_VAR(prefix, 0652); \ + DEF_VAR(prefix, 0653); \ + DEF_VAR(prefix, 0654); \ + DEF_VAR(prefix, 0655); \ + DEF_VAR(prefix, 0656); \ + DEF_VAR(prefix, 0657); \ + DEF_VAR(prefix, 0658); \ + DEF_VAR(prefix, 0659); \ + DEF_VAR(prefix, 0660); \ + DEF_VAR(prefix, 0661); \ + DEF_VAR(prefix, 0662); \ + DEF_VAR(prefix, 0663); \ + DEF_VAR(prefix, 0664); \ + DEF_VAR(prefix, 0665); \ + DEF_VAR(prefix, 0666); \ + DEF_VAR(prefix, 0667); \ + DEF_VAR(prefix, 0668); \ + DEF_VAR(prefix, 0669); \ + DEF_VAR(prefix, 0670); \ + DEF_VAR(prefix, 0671); \ + DEF_VAR(prefix, 0672); \ + DEF_VAR(prefix, 0673); \ + DEF_VAR(prefix, 0674); \ + DEF_VAR(prefix, 0675); \ + DEF_VAR(prefix, 0676); \ + DEF_VAR(prefix, 0677); \ + DEF_VAR(prefix, 0678); \ + DEF_VAR(prefix, 0679); \ + DEF_VAR(prefix, 0680); \ + DEF_VAR(prefix, 0681); \ + DEF_VAR(prefix, 0682); \ + DEF_VAR(prefix, 0683); \ + DEF_VAR(prefix, 0684); \ + DEF_VAR(prefix, 0685); \ + DEF_VAR(prefix, 0686); \ + DEF_VAR(prefix, 0687); \ + DEF_VAR(prefix, 0688); \ + DEF_VAR(prefix, 0689); \ + DEF_VAR(prefix, 0690); \ + DEF_VAR(prefix, 0691); \ + DEF_VAR(prefix, 0692); \ + DEF_VAR(prefix, 0693); \ + DEF_VAR(prefix, 0694); \ + DEF_VAR(prefix, 0695); \ + DEF_VAR(prefix, 0696); \ + DEF_VAR(prefix, 0697); \ + DEF_VAR(prefix, 0698); \ + DEF_VAR(prefix, 0699); \ + DEF_VAR(prefix, 0700); \ + DEF_VAR(prefix, 0701); \ + DEF_VAR(prefix, 0702); \ + DEF_VAR(prefix, 0703); \ + DEF_VAR(prefix, 0704); \ + DEF_VAR(prefix, 0705); \ + DEF_VAR(prefix, 0706); \ + DEF_VAR(prefix, 0707); \ + DEF_VAR(prefix, 0708); \ + DEF_VAR(prefix, 0709); \ + DEF_VAR(prefix, 0710); \ + DEF_VAR(prefix, 0711); \ + DEF_VAR(prefix, 0712); \ + DEF_VAR(prefix, 0713); \ + DEF_VAR(prefix, 0714); \ + DEF_VAR(prefix, 0715); \ + DEF_VAR(prefix, 0716); \ + DEF_VAR(prefix, 0717); \ + DEF_VAR(prefix, 0718); \ + DEF_VAR(prefix, 0719); \ + DEF_VAR(prefix, 0720); \ + DEF_VAR(prefix, 0721); \ + DEF_VAR(prefix, 0722); \ + DEF_VAR(prefix, 0723); \ + DEF_VAR(prefix, 0724); \ + DEF_VAR(prefix, 0725); \ + DEF_VAR(prefix, 0726); \ + DEF_VAR(prefix, 0727); \ + DEF_VAR(prefix, 0728); \ + DEF_VAR(prefix, 0729); \ + DEF_VAR(prefix, 0730); \ + DEF_VAR(prefix, 0731); \ + DEF_VAR(prefix, 0732); \ + DEF_VAR(prefix, 0733); \ + DEF_VAR(prefix, 0734); \ + DEF_VAR(prefix, 0735); \ + DEF_VAR(prefix, 0736); \ + DEF_VAR(prefix, 0737); \ + DEF_VAR(prefix, 0738); \ + DEF_VAR(prefix, 0739); \ + DEF_VAR(prefix, 0740); \ + DEF_VAR(prefix, 0741); \ + DEF_VAR(prefix, 0742); \ + DEF_VAR(prefix, 0743); \ + DEF_VAR(prefix, 0744); \ + DEF_VAR(prefix, 0745); \ + DEF_VAR(prefix, 0746); \ + DEF_VAR(prefix, 0747); \ + DEF_VAR(prefix, 0748); \ + DEF_VAR(prefix, 0749); \ + DEF_VAR(prefix, 0750); \ + DEF_VAR(prefix, 0751); \ + DEF_VAR(prefix, 0752); \ + DEF_VAR(prefix, 0753); \ + DEF_VAR(prefix, 0754); \ + DEF_VAR(prefix, 0755); \ + DEF_VAR(prefix, 0756); \ + DEF_VAR(prefix, 0757); \ + DEF_VAR(prefix, 0758); \ + DEF_VAR(prefix, 0759); \ + DEF_VAR(prefix, 0760); \ + DEF_VAR(prefix, 0761); \ + DEF_VAR(prefix, 0762); \ + DEF_VAR(prefix, 0763); \ + DEF_VAR(prefix, 0764); \ + DEF_VAR(prefix, 0765); \ + DEF_VAR(prefix, 0766); \ + DEF_VAR(prefix, 0767); \ + DEF_VAR(prefix, 0768); \ + DEF_VAR(prefix, 0769); \ + DEF_VAR(prefix, 0770); \ + DEF_VAR(prefix, 0771); \ + DEF_VAR(prefix, 0772); \ + DEF_VAR(prefix, 0773); \ + DEF_VAR(prefix, 0774); \ + DEF_VAR(prefix, 0775); \ + DEF_VAR(prefix, 0776); \ + DEF_VAR(prefix, 0777); \ + DEF_VAR(prefix, 0778); \ + DEF_VAR(prefix, 0779); \ + DEF_VAR(prefix, 0780); \ + DEF_VAR(prefix, 0781); \ + DEF_VAR(prefix, 0782); \ + DEF_VAR(prefix, 0783); \ + DEF_VAR(prefix, 0784); \ + DEF_VAR(prefix, 0785); \ + DEF_VAR(prefix, 0786); \ + DEF_VAR(prefix, 0787); \ + DEF_VAR(prefix, 0788); \ + DEF_VAR(prefix, 0789); \ + DEF_VAR(prefix, 0790); \ + DEF_VAR(prefix, 0791); \ + DEF_VAR(prefix, 0792); \ + DEF_VAR(prefix, 0793); \ + DEF_VAR(prefix, 0794); \ + DEF_VAR(prefix, 0795); \ + DEF_VAR(prefix, 0796); \ + DEF_VAR(prefix, 0797); \ + DEF_VAR(prefix, 0798); \ + DEF_VAR(prefix, 0799); \ + DEF_VAR(prefix, 0800); \ + DEF_VAR(prefix, 0801); \ + DEF_VAR(prefix, 0802); \ + DEF_VAR(prefix, 0803); \ + DEF_VAR(prefix, 0804); \ + DEF_VAR(prefix, 0805); \ + DEF_VAR(prefix, 0806); \ + DEF_VAR(prefix, 0807); \ + DEF_VAR(prefix, 0808); \ + DEF_VAR(prefix, 0809); \ + DEF_VAR(prefix, 0810); \ + DEF_VAR(prefix, 0811); \ + DEF_VAR(prefix, 0812); \ + DEF_VAR(prefix, 0813); \ + DEF_VAR(prefix, 0814); \ + DEF_VAR(prefix, 0815); \ + DEF_VAR(prefix, 0816); \ + DEF_VAR(prefix, 0817); \ + DEF_VAR(prefix, 0818); \ + DEF_VAR(prefix, 0819); \ + DEF_VAR(prefix, 0820); \ + DEF_VAR(prefix, 0821); \ + DEF_VAR(prefix, 0822); \ + DEF_VAR(prefix, 0823); \ + DEF_VAR(prefix, 0824); \ + DEF_VAR(prefix, 0825); \ + DEF_VAR(prefix, 0826); \ + DEF_VAR(prefix, 0827); \ + DEF_VAR(prefix, 0828); \ + DEF_VAR(prefix, 0829); \ + DEF_VAR(prefix, 0830); \ + DEF_VAR(prefix, 0831); \ + DEF_VAR(prefix, 0832); \ + DEF_VAR(prefix, 0833); \ + DEF_VAR(prefix, 0834); \ + DEF_VAR(prefix, 0835); \ + DEF_VAR(prefix, 0836); \ + DEF_VAR(prefix, 0837); \ + DEF_VAR(prefix, 0838); \ + DEF_VAR(prefix, 0839); \ + DEF_VAR(prefix, 0840); \ + DEF_VAR(prefix, 0841); \ + DEF_VAR(prefix, 0842); \ + DEF_VAR(prefix, 0843); \ + DEF_VAR(prefix, 0844); \ + DEF_VAR(prefix, 0845); \ + DEF_VAR(prefix, 0846); \ + DEF_VAR(prefix, 0847); \ + DEF_VAR(prefix, 0848); \ + DEF_VAR(prefix, 0849); \ + DEF_VAR(prefix, 0850); \ + DEF_VAR(prefix, 0851); \ + DEF_VAR(prefix, 0852); \ + DEF_VAR(prefix, 0853); \ + DEF_VAR(prefix, 0854); \ + DEF_VAR(prefix, 0855); \ + DEF_VAR(prefix, 0856); \ + DEF_VAR(prefix, 0857); \ + DEF_VAR(prefix, 0858); \ + DEF_VAR(prefix, 0859); \ + DEF_VAR(prefix, 0860); \ + DEF_VAR(prefix, 0861); \ + DEF_VAR(prefix, 0862); \ + DEF_VAR(prefix, 0863); \ + DEF_VAR(prefix, 0864); \ + DEF_VAR(prefix, 0865); \ + DEF_VAR(prefix, 0866); \ + DEF_VAR(prefix, 0867); \ + DEF_VAR(prefix, 0868); \ + DEF_VAR(prefix, 0869); \ + DEF_VAR(prefix, 0870); \ + DEF_VAR(prefix, 0871); \ + DEF_VAR(prefix, 0872); \ + DEF_VAR(prefix, 0873); \ + DEF_VAR(prefix, 0874); \ + DEF_VAR(prefix, 0875); \ + DEF_VAR(prefix, 0876); \ + DEF_VAR(prefix, 0877); \ + DEF_VAR(prefix, 0878); \ + DEF_VAR(prefix, 0879); \ + DEF_VAR(prefix, 0880); \ + DEF_VAR(prefix, 0881); \ + DEF_VAR(prefix, 0882); \ + DEF_VAR(prefix, 0883); \ + DEF_VAR(prefix, 0884); \ + DEF_VAR(prefix, 0885); \ + DEF_VAR(prefix, 0886); \ + DEF_VAR(prefix, 0887); \ + DEF_VAR(prefix, 0888); \ + DEF_VAR(prefix, 0889); \ + DEF_VAR(prefix, 0890); \ + DEF_VAR(prefix, 0891); \ + DEF_VAR(prefix, 0892); \ + DEF_VAR(prefix, 0893); \ + DEF_VAR(prefix, 0894); \ + DEF_VAR(prefix, 0895); \ + DEF_VAR(prefix, 0896); \ + DEF_VAR(prefix, 0897); \ + DEF_VAR(prefix, 0898); \ + DEF_VAR(prefix, 0899); \ + DEF_VAR(prefix, 0900); \ + DEF_VAR(prefix, 0901); \ + DEF_VAR(prefix, 0902); \ + DEF_VAR(prefix, 0903); \ + DEF_VAR(prefix, 0904); \ + DEF_VAR(prefix, 0905); \ + DEF_VAR(prefix, 0906); \ + DEF_VAR(prefix, 0907); \ + DEF_VAR(prefix, 0908); \ + DEF_VAR(prefix, 0909); \ + DEF_VAR(prefix, 0910); \ + DEF_VAR(prefix, 0911); \ + DEF_VAR(prefix, 0912); \ + DEF_VAR(prefix, 0913); \ + DEF_VAR(prefix, 0914); \ + DEF_VAR(prefix, 0915); \ + DEF_VAR(prefix, 0916); \ + DEF_VAR(prefix, 0917); \ + DEF_VAR(prefix, 0918); \ + DEF_VAR(prefix, 0919); \ + DEF_VAR(prefix, 0920); \ + DEF_VAR(prefix, 0921); \ + DEF_VAR(prefix, 0922); \ + DEF_VAR(prefix, 0923); \ + DEF_VAR(prefix, 0924); \ + DEF_VAR(prefix, 0925); \ + DEF_VAR(prefix, 0926); \ + DEF_VAR(prefix, 0927); \ + DEF_VAR(prefix, 0928); \ + DEF_VAR(prefix, 0929); \ + DEF_VAR(prefix, 0930); \ + DEF_VAR(prefix, 0931); \ + DEF_VAR(prefix, 0932); \ + DEF_VAR(prefix, 0933); \ + DEF_VAR(prefix, 0934); \ + DEF_VAR(prefix, 0935); \ + DEF_VAR(prefix, 0936); \ + DEF_VAR(prefix, 0937); \ + DEF_VAR(prefix, 0938); \ + DEF_VAR(prefix, 0939); \ + DEF_VAR(prefix, 0940); \ + DEF_VAR(prefix, 0941); \ + DEF_VAR(prefix, 0942); \ + DEF_VAR(prefix, 0943); \ + DEF_VAR(prefix, 0944); \ + DEF_VAR(prefix, 0945); \ + DEF_VAR(prefix, 0946); \ + DEF_VAR(prefix, 0947); \ + DEF_VAR(prefix, 0948); \ + DEF_VAR(prefix, 0949); \ + DEF_VAR(prefix, 0950); \ + DEF_VAR(prefix, 0951); \ + DEF_VAR(prefix, 0952); \ + DEF_VAR(prefix, 0953); \ + DEF_VAR(prefix, 0954); \ + DEF_VAR(prefix, 0955); \ + DEF_VAR(prefix, 0956); \ + DEF_VAR(prefix, 0957); \ + DEF_VAR(prefix, 0958); \ + DEF_VAR(prefix, 0959); \ + DEF_VAR(prefix, 0960); \ + DEF_VAR(prefix, 0961); \ + DEF_VAR(prefix, 0962); \ + DEF_VAR(prefix, 0963); \ + DEF_VAR(prefix, 0964); \ + DEF_VAR(prefix, 0965); \ + DEF_VAR(prefix, 0966); \ + DEF_VAR(prefix, 0967); \ + DEF_VAR(prefix, 0968); \ + DEF_VAR(prefix, 0969); \ + DEF_VAR(prefix, 0970); \ + DEF_VAR(prefix, 0971); \ + DEF_VAR(prefix, 0972); \ + DEF_VAR(prefix, 0973); \ + DEF_VAR(prefix, 0974); \ + DEF_VAR(prefix, 0975); \ + DEF_VAR(prefix, 0976); \ + DEF_VAR(prefix, 0977); \ + DEF_VAR(prefix, 0978); \ + DEF_VAR(prefix, 0979); \ + DEF_VAR(prefix, 0980); \ + DEF_VAR(prefix, 0981); \ + DEF_VAR(prefix, 0982); \ + DEF_VAR(prefix, 0983); \ + DEF_VAR(prefix, 0984); \ + DEF_VAR(prefix, 0985); \ + DEF_VAR(prefix, 0986); \ + DEF_VAR(prefix, 0987); \ + DEF_VAR(prefix, 0988); \ + DEF_VAR(prefix, 0989); \ + DEF_VAR(prefix, 0990); \ + DEF_VAR(prefix, 0991); \ + DEF_VAR(prefix, 0992); \ + DEF_VAR(prefix, 0993); \ + DEF_VAR(prefix, 0994); \ + DEF_VAR(prefix, 0995); \ + DEF_VAR(prefix, 0996); \ + DEF_VAR(prefix, 0997); \ + DEF_VAR(prefix, 0998); \ + DEF_VAR(prefix, 0999); \ + DEF_VAR(prefix, 1000); \ + DEF_VAR(prefix, 1001); \ + DEF_VAR(prefix, 1002); \ + DEF_VAR(prefix, 1003); \ + DEF_VAR(prefix, 1004); \ + DEF_VAR(prefix, 1005); \ + DEF_VAR(prefix, 1006); \ + DEF_VAR(prefix, 1007); \ + DEF_VAR(prefix, 1008); \ + DEF_VAR(prefix, 1009); \ + DEF_VAR(prefix, 1010); \ + DEF_VAR(prefix, 1011); \ + DEF_VAR(prefix, 1012); \ + DEF_VAR(prefix, 1013); \ + DEF_VAR(prefix, 1014); \ + DEF_VAR(prefix, 1015); \ + DEF_VAR(prefix, 1016); \ + DEF_VAR(prefix, 1017); \ + DEF_VAR(prefix, 1018); \ + DEF_VAR(prefix, 1019); \ + DEF_VAR(prefix, 1020); \ + DEF_VAR(prefix, 1021); \ + DEF_VAR(prefix, 1022); \ + DEF_VAR(prefix, 1023); \ + DEF_VAR(prefix, 1024); diff --git a/ext/opcache/jit/tls/testing/def.c b/ext/opcache/jit/tls/testing/def.c new file mode 100644 index 0000000000000..f18d577e278a5 --- /dev/null +++ b/ext/opcache/jit/tls/testing/def.c @@ -0,0 +1,33 @@ + +/* _tsrm_ls_cache is defined here */ + +#include +#include +#include + +#ifdef NO_SURPLUS +# include "def-vars.h" +DEF_VARS(def); +#endif + +__thread void* _tsrm_ls_cache; + +size_t tsrm_get_ls_cache_tcb_offset(void) { + return 0; +} + +void zend_accel_error(int type, const char *format, ...) { + if (type < 4) { + va_list ap; + va_start(ap, format); + vprintf(format, ap); + va_end(ap); + } +} + +int test(void); + +int decl(void) { + return test(); +} + diff --git a/ext/opcache/jit/tls/testing/main.c b/ext/opcache/jit/tls/testing/main.c new file mode 100644 index 0000000000000..75d40ff7888ea --- /dev/null +++ b/ext/opcache/jit/tls/testing/main.c @@ -0,0 +1,48 @@ +#include +#include + +#ifdef NO_SURPLUS +# include "def-vars.h" +DEF_VARS(main); +#endif + +__thread int some_tls_var; + +#ifndef DL_DECL +int decl(void); +#endif + +int main(void) { + /* Ensure TLS vars are allocated */ + some_tls_var = 1; + + int (*decl_p)(void); +#ifdef DL_DECL + int flags = RTLD_LAZY | RTLD_GLOBAL; +# ifdef RTLD_DEEPBIND + flags |= RTLD_DEEPBIND; +# endif + void *handle = dlopen("./libdef.so", flags); + if (!handle) { + fprintf(stderr, "dlopen: %s\n", dlerror()); + return 1; + } + + decl_p = (int (*)(void)) dlsym(handle, "decl"); + if (!decl_p) { + fprintf(stderr, "dlsym: %s\n", dlerror()); + return 1; + } +#else + decl_p = decl; +#endif + + int ret = decl_p(); + if (!ret) { + fprintf(stderr, "FAIL\n"); + } else { + fprintf(stderr, "OK\n"); + } + + return !ret; +} diff --git a/ext/opcache/jit/tls/testing/test.sh b/ext/opcache/jit/tls/testing/test.sh new file mode 100755 index 0000000000000..8b9ebc46ce2d5 --- /dev/null +++ b/ext/opcache/jit/tls/testing/test.sh @@ -0,0 +1,228 @@ +#!/bin/sh + +set -e +cd "$(dirname "$0")" + +print_test() { + echo "Testing: $1 (CC=$CC LD=$LD CFLAGS=$CFLAGS)" +} + +exe_def_static_user() { + print_test "TLS var defined in executable, used in same object" + + rm -f main + + $CC $CFLAGS -ggdb3 -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -o user.o -c user.c + $CC $CFLAGS -ggdb3 -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -ggdb3 -o main main.c def.o user.o tls.o + + ./main +} + +exe_def_shared_user() { + print_test "TLS var defined in executable, used in shared library" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -ggdb3 -fPIC -o main main.c def.o -Wl,-rpath,$(pwd) -L. -luser + + ./main +} + +shared_def_static_user() { + print_test "TLS var defined in shared library, used in same object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o user.o tls.o + + $CC $CFLAGS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +shared_def_shared_user() { + print_test "TLS var defined in shared object, used in other shared object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +shared_def_static_user_no_surplus() { + print_test "TLS var defined in shared library, used in same object. Likely no static TLS surplus." + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o tls.o user.o + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +shared_def_shared_user_no_surplus() { + print_test "TLS var defined in shared object, used in other shared object. Likely no static TLS surplus." + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -ggdb3 -fPIC -o main main.c -Wl,-rpath,$(pwd) -L. -ldef + + ./main +} + +dl_def_static_user() { + print_test "TLS var defined in dl()'ed object, used in same object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o user.o tls.o + + $CC $CFLAGS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +dl_def_shared_user() { + print_test "TLS var defined in dl()'ed object, used in other shared object" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +dl_def_static_user_no_surplus() { + print_test "TLS var defined in dl()'ed object, used in same object. Likely no surplus TLS" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o user.o tls.o + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +dl_def_shared_user_no_surplus() { + print_test "TLS var defined in dl()'ed object, used in other shared object. Likely no surplus TLS" + + rm -f main + + $CC $CFLAGS -ggdb3 -fPIC -o tls.o -c $TLSC + $CC $CFLAGS -ggdb3 -fPIC -o user.o -c user.c + $CC $CFLAGS $LDFLAGS -shared -o libuser.so user.o tls.o + + $CC $CFLAGS -DNO_SURPLUS -ggdb3 -fPIC -o def.o -c def.c + $CC $CFLAGS $LDFLAGS -shared -o libdef.so def.o -Wl,-rpath,$(pwd) -L. -luser + + $CC $CFLAGS -DNO_SURPLUS $LDFLAGS -DDL_DECL -ggdb3 -fPIC -o main main.c + + ./main +} + +if [ -z "$TLSC" ]; then + echo "Variable TLSC is not set" >&2 + exit 1 +fi + +root=$(pwd)/../../../../.. + +# Cheap musl detection +if test -f /etc/alpine-release; then + MUSL="$CFLAGS -D__MUSL__" +else + MUSL= +fi + +for CC in clang gcc; do + if [ $CC = gcc ] && [ -f /etc/freebsd-update.conf ]; then + RPATH=-Wl,-rpath,/usr/local/lib/gcc13 + else + RPATH= + fi + case $CC in + gcc) + LDs="" + for l in bdf gold; do + if command -v ld.$l >/dev/null 2>&1; then + LDs="$LDs $l" + fi + done + if [ -z "$LDs" ]; then + LDs=ld + fi + ;; + clang) + LDs="ld" + if command -v ld.lld >/dev/null 2>&1; then + LDs="$LDs lld" + fi + ;; + esac + for LD in $LDs; do + for opt in -O0 -O3; do + CFLAGS="$MACHINE $MUSL $opt -Werror -I$root/ext/opcache -I$root/Zend -I$root" + LDFLAGS="$MACHINE -fuse-ld=$LD $RPATH" + + for pic in "-fPIC" "-fno-PIC -static"; do + CFLAGS="$CFLAGS $pic" exe_def_static_user + done + shared_def_static_user + shared_def_static_user_no_surplus + dl_def_static_user + dl_def_static_user_no_surplus + if [ "$EXTERN_TLS_SUPPORT" = yes ]; then + exe_def_shared_user + shared_def_shared_user + shared_def_shared_user_no_surplus + dl_def_shared_user + dl_def_shared_user_no_surplus + fi + done + done +done + +echo "All OK" >&2 diff --git a/ext/opcache/jit/tls/testing/user.c b/ext/opcache/jit/tls/testing/user.c new file mode 100644 index 0000000000000..c27e608f3f4b1 --- /dev/null +++ b/ext/opcache/jit/tls/testing/user.c @@ -0,0 +1,28 @@ + +/* _tsrm_ls_cache is used / inspected here */ + +#include "../zend_jit_tls.h" + +extern __thread void* _tsrm_ls_cache; + +int test(void) +{ + size_t tcb_offset = 0; + size_t module_index = -1; + size_t module_offset = -1; + + /* Ensure the slot is allocated */ + _tsrm_ls_cache = NULL; + + zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets( + &tcb_offset, &module_index, &module_offset); + + printf("tcb_offset: %zd; module_index: %zd; module_offset: %zd\n", + tcb_offset, module_index, module_offset); + + if (result != SUCCESS) { + return 0; + } + + return zend_jit_tsrm_ls_cache_address(tcb_offset, module_index, module_offset) == &_tsrm_ls_cache; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls.h b/ext/opcache/jit/tls/zend_jit_tls.h new file mode 100644 index 0000000000000..5f90429267256 --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls.h @@ -0,0 +1,40 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#ifndef ZEND_JIT_TLS_H +#define ZEND_JIT_TLS_H + +#include "Zend/zend_types.h" + +#include +#include + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +); + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +); + +#endif /* ZEND_JIT_TLS_H */ diff --git a/ext/opcache/jit/tls/zend_jit_tls_aarch64.c b/ext/opcache/jit/tls/zend_jit_tls_aarch64.c new file mode 100644 index 0000000000000..8427342f66154 --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_aarch64.c @@ -0,0 +1,247 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#include "Zend/zend_portability.h" +#include "Zend/zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +/* https://developer.arm.com/documentation/ddi0602/2025-03/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page- */ +#define AARCH64_ADRP_IMM_MASK 0x60ffffe0 /* bits 30-29, 23-5 */ +#define AARCH64_LDR_UNSIGNED_IMM_MASK 0x003ffc00 /* bits 21-10 */ +#define AARCH64_ADD_IMM_MASK 0x003ffc00 /* bits 21-10 */ +#define AARCH64_MOVZ_IMM_MASK 0x001fffe0 /* bits 20-5 */ +#define AARCH64_MOVZ_HW_MASK 0x00600000 /* bits 22-21 */ +#define AARCH64_MOVK_IMM_MASK 0x001fffe0 /* bits 20-5 */ +#define AARCH64_MOVK_HW_MASK 0x00600000 /* bits 22-21 */ +#define AARCH64_NOP 0xd503201f + +#undef USE_FALLBACK + +#ifdef __MUSL__ + +# define DTV_OFFSET -8 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#elif defined(__FreeBSD__) + +# define DTV_OFFSET 0 +/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */ +# define DTV_INDEX_GAP 1 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */ +typedef struct _tls_descriptor { + void* thunk; + int index; + size_t offset; +} tls_descriptor; + +#elif defined(__GLIBC__) + +# define DTV_OFFSET 0 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; + uintptr_t _; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#else +# define USE_FALLBACK 1 +#endif + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { +#ifdef USE_FALLBACK + return FAILURE; +#else + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + + void *addr; + uint32_t *insn; + void *thread_pointer; + + __asm__ __volatile__( + /* Load thread pointer address */ + "mrs %0, tpidr_el0\n" + /* Load next instruction address */ + "adr %1, .+4\n\t" + /* General Dynamic code sequence as expected by linkers */ + "adrp x0, :tlsdesc:_tsrm_ls_cache\n" + "ldr x1, [x0, #:tlsdesc_lo12:_tsrm_ls_cache]\n" + "add x0, x0, :tlsdesc_lo12:_tsrm_ls_cache\n" + ".tlsdesccall _tsrm_ls_cache\n" + "blr x1\n" + "mrs x8, tpidr_el0\n" + "add %2, x8, x0\n" + : "=r" (thread_pointer), "=r" (insn), "=r" (addr) + : + : "x0", "x1", "x8"); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); + + /* Check if the general dynamic code was relaxed by the linker */ + + // adrp x0, #any + if ((insn[0] & ~AARCH64_ADRP_IMM_MASK) != 0x90000000) { + zend_accel_error(ACCEL_LOG_DEBUG, "adrp insn does not match: 0x%08" PRIx32 "\n", insn[0]); + goto code_changed; + } + + // ldr x1, [x0, #any] + if ((insn[1] & ~AARCH64_LDR_UNSIGNED_IMM_MASK) != 0xf9400001) { + zend_accel_error(ACCEL_LOG_DEBUG, "ldr insn does not match: 0x%08" PRIx32 "\n", insn[1]); + goto code_changed; + } + + // add x0, x0, any + if ((insn[2] & ~AARCH64_ADD_IMM_MASK) != 0x91000000) { + zend_accel_error(ACCEL_LOG_DEBUG, "add insn does not match: 0x%08" PRIx32 "x\n", insn[2]); + goto code_changed; + } + + /* Code is intact, we can extract immediate values */ + + uint64_t adrp_imm = (uint64_t)( ((insn[0] & 0x00ffffe0) >> 3) | ((insn[0] & 0x60000000) >> 29) ) << 12; + uint64_t add_imm = (uint64_t)(insn[2] & AARCH64_ADD_IMM_MASK) >> 10; + uint64_t pc = (uint64_t)insn; + uintptr_t **where = (uintptr_t**)((pc & ~(4096-1)) + adrp_imm + add_imm); + + /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst + * section "Relocations for thread-local storage". + * The first entry holds a pointer to the variable's TLS descriptor resolver + * function and the second entry holds a platform-specific offset or + * pointer. */ + tls_descriptor *tlsdesc = (tls_descriptor*)(where[1]); + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == (uintptr_t)tlsdesc) { + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %p from thread pointer (inferred from tlsdesc)\n", tlsdesc); + *tcb_offset = (uintptr_t)tlsdesc; + return SUCCESS; + } + + *module_index = (tlsdesc->index + DTV_INDEX_GAP) * sizeof(dtv_pointer_t); + *module_offset = tlsdesc->offset; + +# if ZEND_DEBUG + /* We've got the TLS descriptor. Double check: */ + + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + addr = (void*)(((dtv_pointer_t*)((char*)dtv + *module_index))->val + *module_offset); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); +# endif + + zend_accel_error(ACCEL_LOG_DEBUG, "dynamic tls module idx %zu offset %zu (inferred from code)\n", (size_t)tlsdesc->index, tlsdesc->offset); + + return SUCCESS; + +code_changed: + + /* Code was changed by the linker. Check if we recognize the updated code */ + + // movz x0, #0, lsl #16 + if ((insn[0] & ~AARCH64_MOVZ_IMM_MASK) != 0xd2a00000) { + zend_accel_error(ACCEL_LOG_DEBUG, "movz insn does not match: 0x%08" PRIx32 "\n", insn[0]); + return FAILURE; + } + + // movk x0, #0x10 + if ((insn[1] & ~AARCH64_MOVK_IMM_MASK) != 0xf2800000) { + zend_accel_error(ACCEL_LOG_DEBUG, "movk insn does not match: 0x%08" PRIx32 "\n", insn[1]); + return FAILURE; + } + + // nop + for (int i = 0; i < 2; i++) { + if (insn[2+i] != AARCH64_NOP) { + zend_accel_error(ACCEL_LOG_DEBUG, "nop(%d) insn does not match: 0x%08" PRIx32 "\n", i, insn[2+i]); + return FAILURE; + } + } + + /* Extract immediate values */ + + uint64_t movz_imm = (insn[0] & AARCH64_MOVZ_IMM_MASK) >> 5; + uint64_t movz_shift = (((insn[0] & AARCH64_MOVZ_HW_MASK) >> 21) << 4); + uint64_t mozk_imm = (insn[1] & AARCH64_MOVK_IMM_MASK) >> 5; + uint64_t offset = (movz_imm << movz_shift) | mozk_imm; + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) { + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %" PRIxPTR " from thread pointer (inferred from code)\n", offset); + *tcb_offset = offset; + return SUCCESS; + } + + zend_accel_error(ACCEL_LOG_DEBUG, "static tls offset does not match: %" PRIxPTR " (expected %" PRIxPTR ")\n", + offset, ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer)); + + return FAILURE; +#endif +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + char *thread_pointer; + __asm__ __volatile__( + "mrs %0, tpidr_el0\n" + : "=r" (thread_pointer) + ); + + if (tcb_offset) { + return thread_pointer + tcb_offset; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + return (void*)(((dtv_pointer_t*)((char*)dtv + module_index))->val + module_offset); + } + return NULL; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_darwin.c b/ext/opcache/jit/tls/zend_jit_tls_darwin.c new file mode 100644 index 0000000000000..47a2f01a5a0ae --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_darwin.c @@ -0,0 +1,82 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov | + * +----------------------------------------------------------------------+ + */ + +#include "Zend/zend_portability.h" +#include "Zend/zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + +#if defined(__x86_64__) + size_t *ti; + __asm__ __volatile__( + "leaq __tsrm_ls_cache(%%rip),%0" + : "=r" (ti)); + *module_offset = ti[2]; + *module_index = ti[1] * 8; + + return SUCCESS; +#endif + + return FAILURE; +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + +#if defined(__x86_64__) + if (tcb_offset) { + char *addr; + __asm__ __volatile__( + "movq %%gs:(%1), %0\n" + : "=r" (addr) + : "r" (tcb_offset) + ); + return addr; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + char *base; + __asm__ __volatile__( + "movq %%gs:(%1), %0\n" + : "=r" (base) + : "r" (module_index) + ); + return base + module_offset; + } +#endif + + return NULL; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_win.c b/ext/opcache/jit/tls/zend_jit_tls_win.c new file mode 100644 index 0000000000000..23f0c1e79baaf --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_win.c @@ -0,0 +1,64 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov | + * +----------------------------------------------------------------------+ + */ + +#include "Zend/zend_portability.h" +#include "Zend/zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +extern uint32_t _tls_index; +extern char *_tls_start; +extern char *_tls_end; + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ +#ifdef _WIN64 + void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; +#else + void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; +#endif + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + *module_index = _tls_index * sizeof(void*); + *module_offset = offset; + return SUCCESS; + } + tls_mem++; + offset += sizeof(void*); + } + + if (offset >= size) { + zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); + } + + return FAILURE; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_x86.c b/ext/opcache/jit/tls/zend_jit_tls_x86.c new file mode 100644 index 0000000000000..bca46c8f82664 --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_x86.c @@ -0,0 +1,239 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#include "zend_portability.h" +#include "zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" +#include "zend_jit_tls.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +#undef USE_FALLBACK + +#ifdef __MUSL__ + +# define DTV_OFFSET 4 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#elif defined(__FreeBSD__) + +# define DTV_OFFSET 4 +# define DTV_INDEX_GAP 1 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +/* https://github.com/freebsd/freebsd-src/blob/6b94546a7ea2dc593f5765bd5465a8b7bb80c325/libexec/rtld-elf/i386/rtld_machdep.h#L65 */ +typedef struct _tls_descriptor { + unsigned long index; + unsigned long offset; +} tls_descriptor; + +#elif defined(__GLIBC__) + +# define DTV_OFFSET 4 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; + uintptr_t _; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#else +# define USE_FALLBACK 1 +#endif + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { +#ifdef USE_FALLBACK + return FAILURE; +#else + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + + void *t_addr; + unsigned char *code; + void *thread_pointer; + + __asm__ __volatile__( + /* Load next instruction address */ + "call 1f\n" + ".subsection 1\n" + "1:\n" + "movl (%%esp), %%ebx\n" + "movl %%ebx, %%esi\n" + "ret\n" + ".previous\n" + /* General Dynamic code sequence as expected by linkers */ + "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n" + "leal _tsrm_ls_cache@TLSGD(,%%ebx,1), %%eax\n" + "call ___tls_get_addr@PLT\n" + /* Load thread pointer address */ + "movl %%gs:0, %%ebx\n" + : "=a" (t_addr), "=S" (code), "=b" (thread_pointer) + ); + + ZEND_ASSERT(t_addr == &_tsrm_ls_cache); + + /* Check if the general dynamic code was relaxed by the linker */ + + // addl any,%ebx + if (memcmp(&code[0], "\x81\xc3", 2) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[0], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "addl insn does not match: 0x%16" PRIx64 "\n", bytes); + goto code_changed; + } + + // leal any(,%ebx,1),%eax + if (memcmp(&code[6], "\x8d\x04\x1d", 3) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[6], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "leal insn does not match: 0x%16" PRIx64 "\n", bytes); + goto code_changed; + } + + // call any + if (memcmp(&code[13], "\xe8", 1) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[13], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "call insn does not match: 0x%16" PRIx64 "\n", bytes); + goto code_changed; + } + + /* Code is intact, we can extract immediate values */ + + uint32_t addl_imm = ((uint32_t)code[5] << 24) + | ((uint32_t)code[4] << 16) + | ((uint32_t)code[3] << 8) + | ((uint32_t)code[2]); + uint32_t leal_imm = ((uint32_t)code[12] << 24) + | ((uint32_t)code[11] << 16) + | ((uint32_t)code[10] << 8) + | ((uint32_t)code[9]); + + tls_descriptor *tlsdesc = (tls_descriptor*)(leal_imm + addl_imm + (uintptr_t)code); + + *module_index = (tlsdesc->index + DTV_INDEX_GAP) * sizeof(dtv_pointer_t); + *module_offset = tlsdesc->offset; + +# if ZEND_DEBUG + /* We've got the TLS descriptor. Double check: */ + + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + void *addr = (void*)(((dtv_pointer_t*)((char*)dtv + *module_index))->val + *module_offset); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); +# endif + + zend_accel_error(ACCEL_LOG_DEBUG, "dynamic tls module idx %zu offset %zu (inferred from code)\n", + (size_t)tlsdesc->index, (size_t)tlsdesc->offset); + + return SUCCESS; + +code_changed: + + /* Code was changed by the linker. Check if we recognize the updated code */ + + /* + * 81 c3 98 2d 00 00 addl $0x2d98,%ebx + * 65 a1 00 00 00 00 movl %gs:0x0,%eax + * 81 e8 04 00 00 00 subl $0x4,%eax + */ + + // movl %gs:0x0,%eax + if (memcmp(&code[6], "\x65\xa1\x00\x00\x00\x00", 6) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[6], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "movl insn does not match: 0x%16" PRIx64 "\n", bytes); + return FAILURE; + } + + // subl $any,%eax + if (memcmp(&code[12], "\x81\xe8", 2) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[6], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "subl insn does not match: 0x%16" PRIx64 "\n", bytes); + return FAILURE; + } + + /* Extract immediate values */ + + uint32_t offset = -(((uint32_t)code[17] << 24) + | ((uint32_t)code[16] << 16) + | ((uint32_t)code[15] << 8) + | ((uint32_t)code[14])); + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) { + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %" PRIx32 " from thread pointer (inferred from code)\n", offset); + *tcb_offset = offset; + return SUCCESS; + } + + zend_accel_error(ACCEL_LOG_DEBUG, "static tls offset does not match: 0x%" PRIx32 " (expected 0x%" PRIx32 ")\n", + offset, (uint32_t)((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer)); + + return FAILURE; +#endif +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + char *thread_pointer; + __asm__ __volatile__( + "movl %%gs:0, %0\n" + : "=r" (thread_pointer) + ); + + if (tcb_offset) { + return thread_pointer + tcb_offset; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + return (void*)(((dtv_pointer_t*)((char*)dtv + module_index))->val + module_offset); + } + return NULL; +} diff --git a/ext/opcache/jit/tls/zend_jit_tls_x86_64.c b/ext/opcache/jit/tls/zend_jit_tls_x86_64.c new file mode 100644 index 0000000000000..3adeb1ff8064c --- /dev/null +++ b/ext/opcache/jit/tls/zend_jit_tls_x86_64.c @@ -0,0 +1,222 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | https://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Arnaud Le Blanc | + * +----------------------------------------------------------------------+ + */ + +#include "zend_portability.h" +#include "zend_types.h" +#include "TSRM/TSRM.h" +#include "zend_accelerator_debug.h" +#include "zend_jit_tls.h" + +#include +#include + +TSRMLS_CACHE_EXTERN(); + +#undef USE_FALLBACK + +#ifdef __MUSL__ + +# define DTV_OFFSET 8 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#elif defined(__FreeBSD__) + +# define DTV_OFFSET 8 +# define DTV_INDEX_GAP 1 + +typedef struct _dtv_pointer_t { + uintptr_t val; +} dtv_pointer_t; + +/* https://github.com/freebsd/freebsd-src/blob/6b94546a7ea2dc593f5765bd5465a8b7bb80c325/libexec/rtld-elf/amd64/rtld_machdep.h#L65 */ +typedef struct _tls_descriptor { + unsigned long index; + unsigned long offset; +} tls_descriptor; + +#elif defined(__GLIBC__) + +# define DTV_OFFSET 8 +# define DTV_INDEX_GAP 0 + +typedef struct _dtv_pointer_t { + uintptr_t val; + uintptr_t _; +} dtv_pointer_t; + +typedef struct _tls_descriptor { + size_t index; + size_t offset; +} tls_descriptor; + +#else +# define USE_FALLBACK 1 +#endif + +zend_result zend_jit_resolve_tsrm_ls_cache_offsets( + size_t *tcb_offset, + size_t *module_index, + size_t *module_offset +) { +#ifdef USE_FALLBACK + return FAILURE; +#else + *tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (*tcb_offset != 0) { + return SUCCESS; + } + + void *addr; + unsigned char *code; + void *thread_pointer; + + __asm__ __volatile__( + /* Load next instruction address */ + "leaq (%%rip), %%rbx\n" + /* General Dynamic code sequence as expected by linkers */ + ".byte 0x66\n" + "leaq _tsrm_ls_cache@tlsgd(%%rip), %%rdi\n" + ".word 0x6666\n" + "rex64\n" + "call __tls_get_addr\n" + /* Load thread pointer address */ + "movq %%fs:0, %%rsi\n" + : "=a" (addr), "=b" (code), "=S" (thread_pointer) + ); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); + + /* Check if the general dynamic code was relaxed by the linker */ + + // data16 leaq any(%rip),%rdi + if (memcmp(&code[0], "\x66\x48\x8d\x3d", 4) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[0], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "leaq insn does not match: 0x%016" PRIx64 "\n", bytes); + goto code_changed; + } + + // data16 data16 rex.W call any + if (memcmp(&code[8], "\x66\x66\x48\xe8", 4) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[8], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "call insn does not match: 0x%016" PRIx64 "\n", bytes); + goto code_changed; + } + + /* Code is intact, we can extract immediate values */ + + uintptr_t leaq_imm = (uintptr_t)(int32_t)((uint32_t)code[7] << 24) + | ((uint32_t)code[6] << 16) + | ((uint32_t)code[5] << 8) + | ((uint32_t)code[4]); + + tls_descriptor *tlsdesc = (tls_descriptor*)(leaq_imm + (uintptr_t)code + 8 /* leaq */); + + *module_index = ((size_t)tlsdesc->index + DTV_INDEX_GAP) * sizeof(dtv_pointer_t); + *module_offset = (size_t)tlsdesc->offset; + +# if ZEND_DEBUG + /* We've got the TLS descriptor. Double check: */ + + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + addr = (void*)(((dtv_pointer_t*)((char*)dtv + *module_index))->val + *module_offset); + + ZEND_ASSERT(addr == &_tsrm_ls_cache); +# endif + + zend_accel_error(ACCEL_LOG_DEBUG, "dynamic tls module idx %zu offset %zu (inferred from code)\n", + (size_t)tlsdesc->index, (size_t)tlsdesc->offset); + + return SUCCESS; + +code_changed: + + /* Code was changed by the linker. Check if we recognize the updated code */ + + /* + * 64 48 8b 04 25 00 00 00 00 movq %fs:0x0,%rax + * 48 8d 80 f8 ff ff ff leaq -0x8(%rax),%rax + */ + + // movq %fs:0x0,%rax + if (memcmp(&code[0], "\x64\x48\x8b\x04\x25\x00\x00\x00\x00", 9) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[0], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "movq insn does not match: 0x%016" PRIx64 "\n", bytes); + return FAILURE; + } + + // leaq any(%rax),$rax + if (memcmp(&code[9], "\x48\x8d\x80", 3) != 0) { + uint64_t bytes; + memcpy(&bytes, &code[10], 8); + zend_accel_error(ACCEL_LOG_DEBUG, "leaq insn does not match: 0x%016" PRIx64 "\n", bytes); + return FAILURE; + } + + /* Extract immediate values */ + + uintptr_t offset = (uintptr_t)(int32_t)(((uint32_t)code[15] << 24) + | ((uint32_t)code[14] << 16) + | ((uint32_t)code[13] << 8) + | ((uint32_t)code[12])); + + if ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer == offset) { + *tcb_offset = offset; + zend_accel_error(ACCEL_LOG_DEBUG, "static tls at offset %" PRIxPTR " from thread pointer (inferred from code)\n", offset); + return SUCCESS; + } + + zend_accel_error(ACCEL_LOG_DEBUG, "static tls offset does not match: %" PRIxPTR " (expected %" PRIxPTR ")\n", + offset, ((uintptr_t)&_tsrm_ls_cache - (uintptr_t)thread_pointer)); + + return FAILURE; +#endif +} + +/* Used for testing */ +void *zend_jit_tsrm_ls_cache_address( + size_t tcb_offset, + size_t module_index, + size_t module_offset +) { + char *thread_pointer; + __asm__ __volatile__( + "movq %%fs:0, %0\n" + : "=r" (thread_pointer) + ); + + if (tcb_offset) { + return thread_pointer + tcb_offset; + } + if (module_index != (size_t)-1 && module_offset != (size_t)-1) { + dtv_pointer_t *dtv = *(dtv_pointer_t**)((uintptr_t)thread_pointer + DTV_OFFSET); + return (void*)(((dtv_pointer_t*)((char*)dtv + module_index))->val + module_offset); + } + return NULL; +} diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index 6aa7896554920..9aab1262bbe67 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -18,6 +18,7 @@ #include "jit/ir/ir.h" #include "jit/ir/ir_builder.h" +#include "jit/tls/zend_jit_tls.h" #if defined(IR_TARGET_X86) # define IR_REG_SP 4 /* IR_REG_RSP */ @@ -171,15 +172,9 @@ static uint32_t default_mflags = 0; static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ??? #ifdef ZTS -# ifdef _WIN32 -extern uint32_t _tls_index; -extern char *_tls_start; -extern char *_tls_end; -# endif - static size_t tsrm_ls_cache_tcb_offset = 0; -static size_t tsrm_tls_index = 0; -static size_t tsrm_tls_offset = 0; +static size_t tsrm_tls_index = -1; +static size_t tsrm_tls_offset = -1; # define EG_TLS_OFFSET(field) \ (executor_globals_offset + offsetof(zend_executor_globals, field)) @@ -334,6 +329,8 @@ static int zend_jit_assign_to_variable(zend_jit_ctx *jit, zend_jit_addr ref_addr, bool check_exception); +static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags); + typedef struct _zend_jit_stub { const char *name; int (*stub)(zend_jit_ctx *jit); @@ -463,6 +460,11 @@ static const char* zend_reg_name(int8_t reg) /* IR helpers */ #ifdef ZTS +static void * ZEND_FASTCALL zend_jit_get_tsrm_ls_cache(void) +{ + return _tsrm_ls_cache; +} + static ir_ref jit_TLS(zend_jit_ctx *jit) { ZEND_ASSERT(jit->ctx.control); @@ -482,9 +484,15 @@ static ir_ref jit_TLS(zend_jit_ctx *jit) ref = insn->op1; } } - jit->tls = ir_TLS( - tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index, - tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset); + + if (tsrm_ls_cache_tcb_offset == 0 && tsrm_tls_index == -1) { + jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache)); + } else { + jit->tls = ir_TLS( + tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index, + tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset); + } + return jit->tls; } #endif @@ -3127,6 +3135,8 @@ static void zend_jit_setup_disasm(void) REGISTER_DATA(EG(symbol_table)); REGISTER_DATA(CG(map_ptr_base)); +#else /* ZTS */ + REGISTER_HELPER(zend_jit_get_tsrm_ls_cache); #endif #endif } @@ -3293,7 +3303,6 @@ static void zend_jit_setup_unwinder(void) } #endif - static void zend_jit_setup(bool reattached) { #if defined(IR_TARGET_X86) @@ -3312,166 +3321,17 @@ static void zend_jit_setup(bool reattached) } # endif #endif -#ifdef ZTS -#if defined(IR_TARGET_AARCH64) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - -# ifdef __FreeBSD__ - if (tsrm_ls_cache_tcb_offset == 0) { - TLSDescriptor **where; - - __asm__( - "adrp %0, :tlsdesc:_tsrm_ls_cache\n" - "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n" - : "=r" (where)); - /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst - * section "Relocations for thread-local storage". - * The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds - * a platform-specific offset or pointer. */ - TLSDescriptor *tlsdesc = where[1]; - - tsrm_tls_offset = tlsdesc->offset; - /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */ - tsrm_tls_index = (tlsdesc->index + 1) * 8; - } -# elif defined(__MUSL__) - if (tsrm_ls_cache_tcb_offset == 0) { - size_t **where; - - __asm__( - "adrp %0, :tlsdesc:_tsrm_ls_cache\n" - "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n" - : "=r" (where)); - /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst */ - size_t *tlsdesc = where[1]; - - tsrm_tls_offset = tlsdesc[1]; - tsrm_tls_index = tlsdesc[0] * 8; - } -# else - ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); -# endif -# elif defined(_WIN64) - tsrm_tls_index = _tls_index * sizeof(void*); - - /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ - /* Probably, it might be better solution */ - do { - void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; - void *val = _tsrm_ls_cache; - size_t offset = 0; - size_t size = (char*)&_tls_end - (char*)&_tls_start; - - while (offset < size) { - if (*tls_mem == val) { - tsrm_tls_offset = offset; - break; - } - tls_mem++; - offset += sizeof(void*); - } - if (offset >= size) { - zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); - } - } while(0); -# elif defined(ZEND_WIN32) - tsrm_tls_index = _tls_index * sizeof(void*); - - /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ - /* Probably, it might be better solution */ - do { - void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; - void *val = _tsrm_ls_cache; - size_t offset = 0; - size_t size = (char*)&_tls_end - (char*)&_tls_start; - - while (offset < size) { - if (*tls_mem == val) { - tsrm_tls_offset = offset; - break; - } - tls_mem++; - offset += sizeof(void*); - } - if (offset >= size) { - zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size"); - } - } while(0); -# elif defined(__APPLE__) && defined(__x86_64__) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - if (tsrm_ls_cache_tcb_offset == 0) { - size_t *ti; - __asm__( - "leaq __tsrm_ls_cache(%%rip),%0" - : "=r" (ti)); - tsrm_tls_offset = ti[2]; - tsrm_tls_index = ti[1] * 8; - } -# elif defined(__GNUC__) && defined(__x86_64__) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - if (tsrm_ls_cache_tcb_offset == 0) { -#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && \ - !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) - size_t ret; - - asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" - : "=r" (ret)); - tsrm_ls_cache_tcb_offset = ret; -#elif defined(__MUSL__) - size_t *ti; - - __asm__( - "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" - : "=D" (ti)); - tsrm_tls_offset = ti[1]; - tsrm_tls_index = ti[0] * 8; -#elif defined(__FreeBSD__) - size_t *ti; - - __asm__( - "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" - : "=D" (ti)); - tsrm_tls_offset = ti[1]; - /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */ - tsrm_tls_index = (ti[0] + 1) * 8; -#else - size_t *ti; - - __asm__( - "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" - : "=D" (ti)); - tsrm_tls_offset = ti[1]; - tsrm_tls_index = ti[0] * 16; -#endif - } -# elif defined(__GNUC__) && defined(__i386__) - tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); - if (tsrm_ls_cache_tcb_offset == 0) { -#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) - size_t ret; - asm ("leal _tsrm_ls_cache@ntpoff,%0\n" - : "=a" (ret)); - tsrm_ls_cache_tcb_offset = ret; -#else - size_t *ti, _ebx, _ecx, _edx; - - __asm__( - "call 1f\n" - ".subsection 1\n" - "1:\tmovl (%%esp), %%ebx\n\t" - "ret\n" - ".previous\n\t" - "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" - "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" - "call ___tls_get_addr@plt\n\t" - "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" - : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); - tsrm_tls_offset = ti[1]; - tsrm_tls_index = ti[0] * 8; -#endif +#ifdef ZTS + zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets( + &tsrm_ls_cache_tcb_offset, + &tsrm_tls_index, + &tsrm_tls_offset + ); + if (result == FAILURE) { + zend_accel_error(ACCEL_LOG_INFO, + "Could not get _tsrm_ls_cache offsets, will fallback to runtime resolution"); } -# endif #endif #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64) diff --git a/ext/opcache/tests/zzz_basic_logging.phpt b/ext/opcache/tests/zzz_basic_logging.phpt index a53c6a8db9de1..d374daa67ba81 100644 --- a/ext/opcache/tests/zzz_basic_logging.phpt +++ b/ext/opcache/tests/zzz_basic_logging.phpt @@ -19,6 +19,7 @@ opcache --FILE--