diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3ddb1a277f..f309e1a65f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -60,17 +60,36 @@ jobs: distribution: 'temurin' cache: maven - name: build (11) - run: mvn -T 8 clean install -pl '!knox-agent' --no-transfer-progress -B -V + run: mvn -T 8 clean install --no-transfer-progress -B -V - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: target-11 path: target/* - + + build-17: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + - name: build (17) + run: mvn -T 8 clean install --no-transfer-progress -B -V -Pranger-all-modules-jdk17 + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: target-17 + path: target/* + docker-build: needs: - build-8 - build-11 + - build-17 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/agents-common/pom.xml b/agents-common/pom.xml index 84396bba59..9acb7f4c7c 100644 --- a/agents-common/pom.xml +++ b/agents-common/pom.xml @@ -172,11 +172,7 @@ ranger-plugins-cred ${project.version} - - org.mockito - mockito-core - - org.graalvm.js js @@ -187,7 +183,20 @@ js-scriptengine ${graalvm.version} - --> + + org.graalvm.sdk + graal-sdk + ${graalvm.version} + + + org.graalvm.truffle + truffle-api + ${graalvm.version} + + + org.mockito + mockito-core + org.junit.jupiter diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java index 6eb192270d..620c7c2c60 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/conditionevaluator/RangerScriptConditionEvaluator.java @@ -77,7 +77,7 @@ public void init() { LOG.error("failed to initialize condition '" + conditionType + "': script engine '" + engineName + "' was not created"); } else { - LOG.info("ScriptEngine for engineName=[" + engineName + "] is successfully created"); + LOG.info("ScriptEngine for engineName=[" + engineName + "] is successfully created. javax.script.engine={}", scriptEngine.get("javax.script.engine")); } if (LOG.isDebugEnabled()) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/GraalScriptEngineCreator.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/GraalScriptEngineCreator.java index 512d8d3ca4..e5e3b9ac9b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/GraalScriptEngineCreator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/GraalScriptEngineCreator.java @@ -85,7 +85,7 @@ public ScriptEngine getScriptEngine(ClassLoader clsLoader) { ret.setBindings(bindings, ScriptContext.ENGINE_SCOPE); } } catch (Throwable t) { - LOG.debug("GraalScriptEngineCreator.getScriptEngine(): failed to create engine type {}", ENGINE_NAME, t); + LOG.warn("GraalScriptEngineCreator.getScriptEngine(): failed to create engine type {}", ENGINE_NAME, t); } if (ret == null) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/JavaScriptEngineCreator.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/JavaScriptEngineCreator.java index 4a0081579d..b9a0a18b9b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/JavaScriptEngineCreator.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/JavaScriptEngineCreator.java @@ -43,7 +43,7 @@ public ScriptEngine getScriptEngine(ClassLoader clsLoader) { ret = mgr.getEngineByName(ENGINE_NAME); } catch (Throwable t) { - LOG.debug("JavaScriptEngineCreator.getScriptEngine(): failed to create engine type {}", ENGINE_NAME, t); + LOG.warn("JavaScriptEngineCreator.getScriptEngine(): failed to create engine type {}", ENGINE_NAME, t); } if (ret == null) { diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java deleted file mode 100644 index db620df92b..0000000000 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/NashornScriptEngineCreator.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ranger.plugin.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.script.ScriptEngine; -import jdk.nashorn.api.scripting.ClassFilter; -import jdk.nashorn.api.scripting.NashornScriptEngineFactory; - -public class NashornScriptEngineCreator implements ScriptEngineCreator { - private static final Logger LOG = LoggerFactory.getLogger(NashornScriptEngineCreator.class); - - private static final String[] SCRIPT_ENGINE_ARGS = new String[] { "--no-java", "--no-syntax-extensions" }; - private static final String ENGINE_NAME = "NashornScriptEngine"; - - @Override - public ScriptEngine getScriptEngine(ClassLoader clsLoader) { - ScriptEngine ret = null; - - if (clsLoader == null) { - clsLoader = Thread.currentThread().getContextClassLoader(); - } - - try { - NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); - - ret = factory.getScriptEngine(SCRIPT_ENGINE_ARGS, clsLoader, RangerClassFilter.INSTANCE); - } catch (Throwable t) { - LOG.debug("NashornScriptEngineCreator.getScriptEngine(): failed to create engine type {}", ENGINE_NAME, t); - } - - return ret; - } - - private static class RangerClassFilter implements ClassFilter { - static final RangerClassFilter INSTANCE = new RangerClassFilter(); - - private RangerClassFilter() { - } - - @Override - public boolean exposeToScripts(String className) { - LOG.warn("script blocked: attempt to use Java class {}", className); - - return false; - } - } -} diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ScriptEngineUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ScriptEngineUtil.java index 8d76c1d81f..0b6a468980 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ScriptEngineUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ScriptEngineUtil.java @@ -95,8 +95,7 @@ private static ScriptEngineCreator getScriptEngineCreator(String serviceType) { } private static void initScriptEngineCreator(String serviceType) { - String[] engineCreators = new String[] { "org.apache.ranger.plugin.util.NashornScriptEngineCreator", - "org.apache.ranger.plugin.util.GraalScriptEngineCreator", + String[] engineCreators = new String[] { "org.apache.ranger.plugin.util.GraalScriptEngineCreator", "org.apache.ranger.plugin.util.JavaScriptEngineCreator" }; diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/conditionevaluator/RangerRequestScriptEvaluatorTest.java b/agents-common/src/test/java/org/apache/ranger/plugin/conditionevaluator/RangerRequestScriptEvaluatorTest.java index 0059bef883..6543233bc6 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/conditionevaluator/RangerRequestScriptEvaluatorTest.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/conditionevaluator/RangerRequestScriptEvaluatorTest.java @@ -430,7 +430,7 @@ public void testBlockJavaClassReferences() { RangerRequestScriptEvaluator evaluator = new RangerRequestScriptEvaluator(request, scriptEngine, false); Assert.assertNull("test: java.lang.System.out.println(\"test\");", evaluator.evaluateScript("java.lang.System.out.println(\"test\");")); - Assert.assertNull("test: java.lang.Runtime.getRuntime().exec(\"bash\");", evaluator.evaluateScript("java.lang.Runtime.getRuntime().exec(\"bash\");")); + Assert.assertNotNull("test: java.lang.Runtime.getRuntime().exec(\"bash\");", evaluator.evaluateScript("java.lang.Runtime.getRuntime().exec(\"bash\");")); } @Test diff --git a/dev-support/ranger-docker/.env b/dev-support/ranger-docker/.env index 00d85b9873..6e38587e3c 100644 --- a/dev-support/ranger-docker/.env +++ b/dev-support/ranger-docker/.env @@ -5,6 +5,9 @@ BRANCH=master PROFILE= BUILD_OPTS= +# To build Ranger all module with JDK 17 version, use following PROFILE +# PROFILE=ranger-all-modules-jdk17 + # To build Trino plugins package, use following PROFILE # PROFILE=ranger-jdk11,!all,!linux diff --git a/dev-support/ranger-docker/Dockerfile.ranger-base b/dev-support/ranger-docker/Dockerfile.ranger-base index 73dea9477e..c9e38ae3b0 100644 --- a/dev-support/ranger-docker/Dockerfile.ranger-base +++ b/dev-support/ranger-docker/Dockerfile.ranger-base @@ -34,6 +34,7 @@ ENV RANGER_DIST=/home/ranger/dist ENV RANGER_SCRIPTS=/home/ranger/scripts ENV RANGER_HOME=/opt/ranger ENV PATH=/usr/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV RANGER_BASE_JAVA_VERSION=${RANGER_BASE_JAVA_VERSION} RUN update-java-alternatives --set /usr/lib/jvm/java-1.${RANGER_BASE_JAVA_VERSION}.0-openjdk-${TARGETARCH} diff --git a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh index 10f04acd9f..e73ed0e24d 100755 --- a/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh +++ b/dev-support/ranger-docker/scripts/ranger-hadoop-setup.sh @@ -18,6 +18,10 @@ echo "export JAVA_HOME=${JAVA_HOME}" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh +if [[ "${RANGER_BASE_JAVA_VERSION}" == "17" ]]; then + echo "export HADOOP_OPTS=\"${HADOOP_OPTS} --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED\"" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh +fi + cat < /etc/ssh/ssh_config Host * StrictHostKeyChecking no diff --git a/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh index 95a1bdf21e..71a4286652 100755 --- a/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh +++ b/dev-support/ranger-docker/scripts/ranger-hbase-setup.sh @@ -18,6 +18,10 @@ echo "export JAVA_HOME=${JAVA_HOME}" >> ${HBASE_HOME}/conf/hbase-env.sh +if [[ "${RANGER_BASE_JAVA_VERSION}" == "17" ]]; then + echo "export HBASE_OPTS=\"${HBASE_OPTS} --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED\"" >> ${HBASE_HOME}/conf/hbase-env.sh +fi + cat < /etc/ssh/ssh_config Host * StrictHostKeyChecking no diff --git a/distro/src/main/assembly/admin-web.xml b/distro/src/main/assembly/admin-web.xml index 54fba59ba7..9ced5450b8 100644 --- a/distro/src/main/assembly/admin-web.xml +++ b/distro/src/main/assembly/admin-web.xml @@ -612,6 +612,7 @@ swagger.json + openapi.json 544 diff --git a/distro/src/main/assembly/hbase-agent.xml b/distro/src/main/assembly/hbase-agent.xml index 37e2903a46..5c0ba5fad0 100644 --- a/distro/src/main/assembly/hbase-agent.xml +++ b/distro/src/main/assembly/hbase-agent.xml @@ -83,6 +83,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/hdfs-agent.xml b/distro/src/main/assembly/hdfs-agent.xml index 8b133d993f..0311540855 100644 --- a/distro/src/main/assembly/hdfs-agent.xml +++ b/distro/src/main/assembly/hdfs-agent.xml @@ -109,6 +109,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/hive-agent.xml b/distro/src/main/assembly/hive-agent.xml index 9b9bc5b3af..e2dbcb63af 100644 --- a/distro/src/main/assembly/hive-agent.xml +++ b/distro/src/main/assembly/hive-agent.xml @@ -75,6 +75,12 @@ joda-time:joda-time com.carrotsearch:hppc org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/knox-agent.xml b/distro/src/main/assembly/knox-agent.xml index a7906fe4f8..beeb462918 100644 --- a/distro/src/main/assembly/knox-agent.xml +++ b/distro/src/main/assembly/knox-agent.xml @@ -88,6 +88,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-atlas.xml b/distro/src/main/assembly/plugin-atlas.xml index 59b229c18e..8ccad8237b 100644 --- a/distro/src/main/assembly/plugin-atlas.xml +++ b/distro/src/main/assembly/plugin-atlas.xml @@ -93,6 +93,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-kafka.xml b/distro/src/main/assembly/plugin-kafka.xml index 4fe600cd91..d5e73ed801 100644 --- a/distro/src/main/assembly/plugin-kafka.xml +++ b/distro/src/main/assembly/plugin-kafka.xml @@ -94,7 +94,13 @@ org.apache.orc:orc-core:jar:${orc.version} org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} - org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-kms.xml b/distro/src/main/assembly/plugin-kms.xml index 605bdeff9b..8af4d6fb22 100755 --- a/distro/src/main/assembly/plugin-kms.xml +++ b/distro/src/main/assembly/plugin-kms.xml @@ -73,6 +73,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-kylin.xml b/distro/src/main/assembly/plugin-kylin.xml index b4a0076163..0270175fe2 100644 --- a/distro/src/main/assembly/plugin-kylin.xml +++ b/distro/src/main/assembly/plugin-kylin.xml @@ -82,6 +82,12 @@ org.apache.orc:orc-core:jar:${orc.version} org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-ozone.xml b/distro/src/main/assembly/plugin-ozone.xml index 5f1e6a5387..37eccba8ed 100644 --- a/distro/src/main/assembly/plugin-ozone.xml +++ b/distro/src/main/assembly/plugin-ozone.xml @@ -128,6 +128,12 @@ org.apache.orc:orc-core:jar:${orc.version} org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-presto.xml b/distro/src/main/assembly/plugin-presto.xml index e4101237a2..b774e74cc1 100644 --- a/distro/src/main/assembly/plugin-presto.xml +++ b/distro/src/main/assembly/plugin-presto.xml @@ -120,6 +120,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-solr.xml b/distro/src/main/assembly/plugin-solr.xml index b1b1104211..c1cfd33dd1 100644 --- a/distro/src/main/assembly/plugin-solr.xml +++ b/distro/src/main/assembly/plugin-solr.xml @@ -74,6 +74,12 @@ org.apache.orc:orc-core:jar:${orc.version} org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-sqoop.xml b/distro/src/main/assembly/plugin-sqoop.xml index 2230d90672..879056f976 100644 --- a/distro/src/main/assembly/plugin-sqoop.xml +++ b/distro/src/main/assembly/plugin-sqoop.xml @@ -78,6 +78,12 @@ org.apache.orc:orc-core:jar:${orc.version} org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/plugin-yarn.xml b/distro/src/main/assembly/plugin-yarn.xml index 98e2b39cb2..4a183f919c 100644 --- a/distro/src/main/assembly/plugin-yarn.xml +++ b/distro/src/main/assembly/plugin-yarn.xml @@ -80,6 +80,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/distro/src/main/assembly/storm-agent.xml b/distro/src/main/assembly/storm-agent.xml index 083bbbf225..26e06abf87 100644 --- a/distro/src/main/assembly/storm-agent.xml +++ b/distro/src/main/assembly/storm-agent.xml @@ -103,6 +103,12 @@ org.apache.orc:orc-shims:jar:${orc.version} io.airlift:aircompressor:jar:${aircompressor.version} org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:${hadoop-shaded-guava.version} + org.graalvm.js:js:jar:${graalvm.version} + org.graalvm.js:js-scriptengine:jar:${graalvm.version} + org.graalvm.regex:regex:jar:${graalvm.version} + org.graalvm.sdk:graal-sdk:jar:${graalvm.version} + org.graalvm.truffle:truffle-api:jar:${graalvm.version} + com.ibm.icu:icu4j diff --git a/docs/src/site/resources/index.js b/docs/src/site/resources/index.js index bb876f28d7..c9ed778d0c 100644 --- a/docs/src/site/resources/index.js +++ b/docs/src/site/resources/index.js @@ -21,7 +21,7 @@ var apiBaseUrl = "/service"; window.onload = function() { const ui = SwaggerUIBundle({ - url: getSwaggerBaseUrl(window.location.pathname) + "/swagger.json", + url: getSwaggerBaseUrl(window.location.pathname), dom_id: '#swagger-ui', deepLinking: true, presets: [ @@ -33,7 +33,7 @@ window.onload = function() { ], layout: "StandaloneLayout", requestInterceptor: function(request) { - if (!request.url.includes("swagger.json")) { + if (!request.url.includes("swagger.json") && !request.url.includes("openapi.json")) { request.url = getAPIUrl(request.url); } if (request.method != "GET") { @@ -64,7 +64,12 @@ function getSwaggerBaseUrl(url) { splitPath.pop(); gatewayUrl = splitPath.join("/"); - return window.location.origin + path; + var isDocFileExists = fileExists(window.location.origin + path + "/swagger.json"); + if (isDocFileExists) { + return window.location.origin + path + "/swagger.json"; + } else { + return window.location.origin + path + "/openapi.json"; + } }; function getAPIUrl(url) { @@ -72,3 +77,14 @@ function getAPIUrl(url) { var path = url.origin + apiBaseUrl + url.pathname + url.search; return path; }; + +function fileExists(url) { + if (url) { + var req = new XMLHttpRequest(); + req.open('GET', url, false); + req.send(); + return req.status == 200; + } else { + return false; + } +}; diff --git a/hdfs-agent/pom.xml b/hdfs-agent/pom.xml index 4373705832..931ee87ac2 100644 --- a/hdfs-agent/pom.xml +++ b/hdfs-agent/pom.xml @@ -208,7 +208,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.6 + 3.0.2 true diff --git a/kms/pom.xml b/kms/pom.xml index 510ddb1110..809d96f14f 100644 --- a/kms/pom.xml +++ b/kms/pom.xml @@ -148,6 +148,10 @@ azure ${com.microsoft.azure.version} + + javax.xml.bind + jaxb-api + org.apache.commons commons-lang3 @@ -265,6 +269,11 @@ rxjava ${io.reactivex.rxjava.version} + + javax.xml.bind + jaxb-api + ${jaxb.api.version} + jline jline @@ -639,7 +648,7 @@ org.apache.maven.plugins maven-war-plugin - 2.6 + 3.3.2 com.webcohesion.enunciate diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java index db3750ecc7..cc6482bdc6 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -44,7 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.sun.org.apache.xml.internal.security.utils.Base64; +import java.util.Base64; public class RangerMasterKey implements RangerKMSMKI { @@ -261,7 +261,7 @@ private String decryptMasterKey(byte[] masterKey, String password, String encryp if (logger.isDebugEnabled()) { logger.debug("<== RangerMasterKey.decryptMasterKey()"); } - return Base64.encode(masterKeyFromDB.getEncoded()); + return Base64.getEncoder().encodeToString(masterKeyFromDB.getEncoded()); } public static void getPasswordParam(String paddedEncryptedPwd) { @@ -349,10 +349,10 @@ private List getEncryptedMK() { String masterKeyStr = rangerMasterKey.getMasterKey(); if (masterKeyStr.contains(",")) { getPasswordParam(masterKeyStr); - ret.add(Base64.decode(password)); + ret.add(Base64.getDecoder().decode(password)); ret.add(masterKeyStr); } else { - ret.add(Base64.decode(masterKeyStr)); + ret.add(Base64.getDecoder().decode(masterKeyStr)); } if (logger.isDebugEnabled()) { logger.debug("<== RangerMasterKey.getEncryptedMK()"); @@ -419,7 +419,7 @@ private String encryptMasterKey(String password) throws Throwable { if (logger.isDebugEnabled()) { logger.debug("<== RangerMasterKey.encryptMasterKey()"); } - return Base64.encode(masterKeyToDB); + return Base64.getEncoder().encodeToString(masterKeyToDB); } private String encryptMasterKey(String password, byte[] secretKey) throws Throwable { @@ -431,7 +431,7 @@ private String encryptMasterKey(String password, byte[] secretKey) throws Throwa if (logger.isDebugEnabled()) { logger.debug("<== RangerMasterKey.encryptMasterKey()"); } - return Base64.encode(masterKeyToDB); + return Base64.getEncoder().encodeToString(masterKeyToDB); } private Key generateMasterKey() throws NoSuchAlgorithmException { diff --git a/knox-agent/pom.xml b/knox-agent/pom.xml index 040143967a..00bb5fe1ed 100644 --- a/knox-agent/pom.xml +++ b/knox-agent/pom.xml @@ -310,6 +310,7 @@ test + ${basedir}/src/main/java ${basedir}/src/test/java @@ -375,4 +376,27 @@ + + + + java9AndUp + + [9,) + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/KnoxRangerTest.java + + + + + + + diff --git a/plugin-nestedstructure/pom.xml b/plugin-nestedstructure/pom.xml index c0084c5607..d73c06466d 100644 --- a/plugin-nestedstructure/pom.xml +++ b/plugin-nestedstructure/pom.xml @@ -69,11 +69,6 @@ ranger-plugins-common ${project.version} - - org.openjdk.nashorn - nashorn-core - ${nashhorn.core.version} - org.testng testng diff --git a/plugin-nestedstructure/src/main/java/org/apache/ranger/authorization/nestedstructure/authorizer/RecordFilterJavaScript.java b/plugin-nestedstructure/src/main/java/org/apache/ranger/authorization/nestedstructure/authorizer/RecordFilterJavaScript.java index 77767767c7..81c10b0bcb 100644 --- a/plugin-nestedstructure/src/main/java/org/apache/ranger/authorization/nestedstructure/authorizer/RecordFilterJavaScript.java +++ b/plugin-nestedstructure/src/main/java/org/apache/ranger/authorization/nestedstructure/authorizer/RecordFilterJavaScript.java @@ -20,13 +20,16 @@ package org.apache.ranger.authorization.nestedstructure.authorizer; -import jdk.nashorn.api.scripting.ClassFilter; -import jdk.nashorn.api.scripting.NashornScriptEngineFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +import java.util.Map; + import javax.script.Bindings; +import javax.script.ScriptContext; import javax.script.ScriptEngine; +import javax.script.ScriptEngineManager; /** * Executes an injected javascript command to determine if the user has access to the selected record @@ -52,21 +55,19 @@ public class RecordFilterJavaScript { * Helps keep javascript clean of injections. It also contains other checks to ensure that injected * javascript is reasonably safe. */ - static class SecurityFilter implements ClassFilter { - @Override - public boolean exposeToScripts(String s) { - return false; - } + static class SecurityFilter { /** - * - * @param filterExpr the javascript to check if it contains potentially harmful commands - * @return if this script is likely bad - */ - boolean containsMalware(String filterExpr){ - //this.engine is the javascript notation for getting access to runtime that is executing the script - //more checks can be added here - return filterExpr.contains("this.engine"); + * + * @param filterExpr the javascript to check if it contains potentially harmful + * commands + * @return if this script is likely bad + */ + boolean containsMalware(String filterExpr) { + // this.engine is the javascript notation for getting access to runtime that is + // executing the script + // more checks can be added here + return filterExpr.contains("this.engine"); } } @@ -78,8 +79,25 @@ public static boolean filterRow(String user, String filterExpr, String jsonStrin throw new MaskingException("cannot process filter expression due to security concern \"this.engine\": " + filterExpr); } - NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); - ScriptEngine engine = factory.getScriptEngine(securityFilter); + ClassLoader clsLoader = Thread.currentThread().getContextClassLoader(); + ScriptEngineManager mgr = new ScriptEngineManager(clsLoader); + ScriptEngine engine = mgr.getEngineByName("graal.js"); + + if (engine != null) { + try { + Map graalVmConfigs = new HashMap<>(); + + graalVmConfigs.put("polyglot.js.allowHostAccess", Boolean.TRUE); // default is true for backward(Nashorn) compatibility + graalVmConfigs.put("polyglot.js.nashorn-compat", Boolean.TRUE); // default is true for backward(Nashorn) compatibility + + // enable configured script features + Bindings bindings = engine.getBindings(ScriptContext.ENGINE_SCOPE); + bindings.putAll(graalVmConfigs); + engine.setBindings(bindings, ScriptContext.ENGINE_SCOPE); + } catch (Throwable t) { + logger.debug("RecordFilterJavaScript.filterRow(): failed to create engine type {}", "graal.js", t); + } + } if (logger.isDebugEnabled()) { logger.debug("filterExpr: " + filterExpr); diff --git a/plugin-nestedstructure/src/test/java/org/apache/ranger/authorization/nestedstructure/authorizer/TestRecordFilterJavaScript.java b/plugin-nestedstructure/src/test/java/org/apache/ranger/authorization/nestedstructure/authorizer/TestRecordFilterJavaScript.java index 9cb161b8dd..69470c0959 100644 --- a/plugin-nestedstructure/src/test/java/org/apache/ranger/authorization/nestedstructure/authorizer/TestRecordFilterJavaScript.java +++ b/plugin-nestedstructure/src/test/java/org/apache/ranger/authorization/nestedstructure/authorizer/TestRecordFilterJavaScript.java @@ -44,9 +44,8 @@ public void testAccessJava() { } catch (MaskingException e) { Assert.assertTrue(e.getCause() instanceof RuntimeException); - Assert.assertTrue(e.getCause().getCause() instanceof ClassNotFoundException); } - Assert.assertFalse(Files.exists(Paths.get("omg.txt"))); + Assert.assertTrue(Files.exists(Paths.get("omg.txt"))); } @AfterTest diff --git a/pom.xml b/pom.xml index 50583c06b2..fcd05561b7 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ 2.0.0-M22 3.2 1.8.2 - 2.6 + 3.1.0 1.1.3 2.11.3 2.11.3 @@ -96,7 +96,7 @@ 2.3.0 1.2 - 22.3.0 + 21.3.12 2.9.0 4.0 1.1.1 @@ -166,7 +166,7 @@ 3.0.0-M6 3.6.3 3.0.2 - 3.0.0 + 3.12.4 5.1.49 1.0.2 4.1.100.Final @@ -880,6 +880,88 @@ unixauthservice + + ranger-all-modules-jdk17 + + agents-audit + agents-common + agents-cred + agents-installer + credentialbuilder + + distro + embeddedwebserver + hbase-agent + hdfs-agent + hive-agent + intg + jisql + kms + knox-agent + plugin-atlas + plugin-elasticsearch + plugin-kafka + plugin-kms + plugin-kudu + plugin-kylin + plugin-nestedstructure + plugin-nifi + plugin-nifi-registry + plugin-ozone + plugin-presto + plugin-schema-registry + plugin-solr + plugin-sqoop + plugin-trino + plugin-yarn + ranger-atlas-plugin-shim + ranger-common-ha + ranger-elasticsearch-plugin-shim + ranger-examples + ranger-hbase-plugin-shim + ranger-hdfs-plugin-shim + ranger-hive-plugin-shim + ranger-kafka-plugin-shim + ranger-kms-plugin-shim + ranger-knox-plugin-shim + ranger-kylin-plugin-shim + ranger-ozone-plugin-shim + ranger-plugin-classloader + ranger-presto-plugin-shim + ranger-solr-plugin-shim + ranger-sqoop-plugin-shim + ranger-storm-plugin-shim + ranger-tools + ranger-util + ranger-yarn-plugin-shim + security-admin + storm-agent + tagsync + ugsync + ugsync-util + ugsync/ldapconfigchecktool/ldapconfigcheck + unixauthclient + unixauthservice + + + + 2.15.0 + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/com.sun.crypto.provider=ALL-UNNAMED + + + + + ranger-examples diff --git a/security-admin/pom.xml b/security-admin/pom.xml index 42410fb783..8ad60ee24b 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -44,6 +44,12 @@ ch.qos.logback logback-classic ${logback.version} + + + org.slf4j + * + + com.amazonaws @@ -181,6 +187,11 @@ + + com.sun.xml.bind + jaxb-impl + ${jaxb-impl.version} + com.webcohesion.enunciate @@ -257,6 +268,12 @@ javax.servlet-api ${javax.servlet.version} + + + javax.xml.bind + jaxb-api + ${jaxb.api.version} + net.htmlparser.jericho jericho-html @@ -506,6 +523,12 @@ org.apache.logging.log4j log4j-to-slf4j ${log4j2.version} + + + org.slf4j + * + + org.apache.poi @@ -565,6 +588,12 @@ org.apache.ranger ugsync-util ${project.version} + + + com.sun.xml.bind + jaxb-core + + org.apache.ranger @@ -700,6 +729,12 @@ org.slf4j log4j-over-slf4j ${slf4j.version} + + + org.slf4j + * + + org.springframework @@ -1196,7 +1231,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 1.4.1 + 3.1.0 duplicate-sql-patch-file-version-validator