Compiled dag

.buildkite/build.rayci.yml

@@ -77,8 +77,7 @@ steps:
   - label: ":tapioca: build: doc"
     instance_type: medium
     commands:
-      - cd doc
-      - FAST=True make html
+      - FAST=True make -C doc/ html
     depends_on: docbuild
     job_env: docbuild
 
@@ -90,7 +89,7 @@ steps:
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
-        --platform cu11.5.2 --platform cu11.6.2 --platform cu11.7.1 
+        --platform cu11.5.2 --platform cu11.6.2 --platform cu11.7.1
         --platform cu11.8.0 --platform cu12.1.1 --platform cpu
         --image-type ray
         --upload
@@ -114,7 +113,7 @@ steps:
     instance_type: medium-arm64
     commands:
       - bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
-        --platform cu11.5.2 --platform cu11.6.2 --platform cu11.7.1 
+        --platform cu11.5.2 --platform cu11.6.2 --platform cu11.7.1
         --platform cu11.8.0 --platform cu12.1.1 --platform cpu
         --image-type ray
         --architecture aarch64

.buildkite/core.rayci.yml

@@ -45,8 +45,10 @@ steps:
     tags:
       - python
       - oss
+      - skip-on-premerge
     instance_type: medium
     commands:
+      - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
       - (cd dashboard/client && npm ci && npm run build)
       - pip install -e python[client]
       - bazel test --config=ci --jobs=1 $(./ci/run/bazel_export_options)

.buildkite/lint.rayci.yml

@@ -1,63 +1,94 @@
 group: lint
-depends_on:
-  - forge
 steps:
   - label: ":lint-roller: lint: clang format"
+    depends_on:
+      - forge
     commands:
       - pip install -c python/requirements_compiled.txt clang-format
       - ./ci/lint/check-git-clang-format-output.sh
 
   - label: ":lint-roller: lint: code format"
+    depends_on:
+      - forge
     commands:
       - pip install -c python/requirements_compiled.txt -r python/requirements/lint-requirements.txt
       - FORMAT_SH_PRINT_DIFF=1 ./ci/lint/format.sh --all-scripts
 
   - label: ":lint-roller: lint: untested code snippet"
+    depends_on:
+      - forge
     commands:
       - pip install -c python/requirements_compiled.txt semgrep
       - semgrep ci --config semgrep.yml
 
   - label: ":lint-roller: lint: banned words"
+    depends_on:
+      - forge
     commands:
       - ./ci/lint/check-banned-words.sh
 
   - label: ":lint-roller: lint: doc readme"
+    depends_on:
+      - forge
     commands:
       - pip install -c python/requirements_compiled.txt docutils
       - cd python && python setup.py check --restructuredtext --strict --metadata
 
   - label: ":lint-roller: lint: dashboard format"
+    depends_on:
+      - forge
     commands:
       - ./ci/lint/check-dashboard-format.sh
 
   - label: ":lint-roller: lint: copyright format"
+    depends_on:
+      - forge
     commands:
       - ./ci/lint/copyright-format.sh -c
 
   - label: ":lint-roller: lint: bazel team"
+    depends_on:
+      - forge
     commands:
       - bazel query 'kind("cc_test", //...)' --output=xml | python ./ci/lint/check-bazel-team-owner.py
       - bazel query 'kind("py_test", //...)' --output=xml | python ./ci/lint/check-bazel-team-owner.py
 
   - label: ":lint-roller: lint: bazel buildifier"
+    depends_on:
+      - forge
     commands:
       - ./ci/lint/check-bazel-buildifier.sh
 
   - label: ":lint-roller: lint: pytest format"
+    depends_on:
+      - forge
     commands:
       - pip install -c python/requirements_compiled.txt yq
       - ./ci/lint/check-pytest-format.sh
 
   - label: ":lint-roller: lint: test coverage"
+    depends_on:
+      - forge
     commands:
       - python ci/pipeline/check-test-run.py
 
   - label: ":lint-roller: lint: api annotations"
+    depends_on:
+      - forge
     instance_type: medium
     commands:
       - RAY_DISABLE_EXTRA_CPP=1 pip install -e python/[all]
       - ./ci/lint/check_api_annotations.py
 
   - label: ":lint-roller: lint: documentation style"
+    depends_on:
+      - forge
     commands:
       - ./ci/lint/check-documentation-style.sh
+
+  - label: ":lint-roller: lint: linkcheck"
+    instance_type: medium
+    command: make -C doc/ linkcheck
+    depends_on: docbuild
+    job_env: docbuild
+    tags: skip-on-premerge

.buildkite/pipeline.macos.yml

@@ -24,7 +24,7 @@ prelude_commands: &prelude_commands |-
 epilogue_commands: &epilogue_commands |-
   # Persist ray logs
   mkdir -p /tmp/artifacts/.ray/
-  tar -cvzf /tmp/artifacts/.ray/logs.tgz /tmp/ray
+  tar -czf /tmp/artifacts/.ray/logs.tgz /tmp/ray
   # Cleanup runtime environment to save storage
   rm -rf /tmp/ray || true
   # Cleanup local caches - this should not clean up global disk cache

BUILD.bazel

@@ -1537,6 +1537,19 @@ ray_cc_test(
     ],
 )
 
+ray_cc_test(
+    name = "core_worker_client_pool_test",
+    size = "small",
+    srcs = [
+        "src/ray/rpc/worker/test/core_worker_client_pool_test.cc",
+    ],
+    tags = ["team:core"],
+    deps = [
+        ":worker_rpc",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 ray_cc_test(
     name = "gcs_server_rpc_test",
     size = "small",

ci/ray_ci/BUILD.bazel

@@ -92,7 +92,6 @@ py_test(
     ],
 )
 
-
 py_test(
     name = "test_utils",
     size = "small",

ci/ray_ci/rllib.tests.yml

@@ -8,6 +8,7 @@ flaky_tests:
   - //rllib:learning_tests_two_step_game_qmix
   - //rllib:learning_tests_pendulum_cql
   - //rllib:learning_tests_two_step_game_qmix_no_mixer
+  - //rllib:learning_tests_cartpole_crashing_and_stalling_appo
   # algorithm, model and other tests
   - //rllib:test_bc 
   - //rllib:test_a3c

ci/ray_ci/test_tester.py

@@ -7,6 +7,7 @@
 import pytest
 
 from ci.ray_ci.linux_tester_container import LinuxTesterContainer
+from ci.ray_ci.windows_tester_container import WindowsTesterContainer
 from ci.ray_ci.tester import (
     _add_default_except_tags,
     _get_container,
@@ -34,12 +35,33 @@ def test_get_container() -> None:
     with mock.patch(
         "ci.ray_ci.linux_tester_container.LinuxTesterContainer.install_ray",
         return_value=None,
+    ), mock.patch(
+        "ci.ray_ci.windows_tester_container.WindowsTesterContainer.install_ray",
+        return_value=None,
     ):
-        container = _get_container("core", 3, 1, 2, 0)
+        container = _get_container(
+            team="core",
+            operating_system="linux",
+            workers=3,
+            worker_id=1,
+            parallelism_per_worker=2,
+            gpus=0,
+        )
+        assert isinstance(container, LinuxTesterContainer)
         assert container.docker_tag == "corebuild"
         assert container.shard_count == 6
         assert container.shard_ids == [2, 3]
 
+        container = _get_container(
+            team="serve",
+            operating_system="windows",
+            workers=3,
+            worker_id=1,
+            parallelism_per_worker=2,
+            gpus=0,
+        )
+        assert isinstance(container, WindowsTesterContainer)
+
 
 def test_get_test_targets() -> None:
     _TEST_YAML = "flaky_tests: [//python/ray/tests:flaky_test_01]"

ci/ray_ci/tester.py

@@ -13,6 +13,7 @@
     DEFAULT_ARCHITECTURE,
 )
 from ci.ray_ci.linux_tester_container import LinuxTesterContainer
+from ci.ray_ci.windows_tester_container import WindowsTesterContainer
 from ci.ray_ci.tester_container import TesterContainer
 from ci.ray_ci.utils import docker_login
 
@@ -135,12 +136,19 @@
     ),
     default="optimized",
 )
+@click.option(
+    "--operating-system",
+    default="linux",
+    type=click.Choice(["linux", "windows"]),
+    help=("Operating system to run tests on"),
+)
 def main(
     targets: List[str],
     team: str,
     workers: int,
     worker_id: int,
     parallelism_per_worker: int,
+    operating_system: str,
     except_tags: str,
     only_tags: str,
     run_flaky_tests: bool,
@@ -155,14 +163,18 @@ def main(
     if not bazel_workspace_dir:
         raise Exception("Please use `bazelisk run //ci/ray_ci`")
     os.chdir(bazel_workspace_dir)
-    docker_login(_DOCKER_ECR_REPO.split("/")[0])
+    # TODO(can): only linux uses intermediate dockers publised to ECR; remove this when
+    # we can build intermediate dockers for Windows
+    if operating_system == "linux":
+        docker_login(_DOCKER_ECR_REPO.split("/")[0])
 
     if build_type == "wheel" or build_type == "wheel-aarch64":
         # for wheel testing, we first build the wheel and then use it for running tests
         architecture = DEFAULT_ARCHITECTURE if build_type == "wheel" else "aarch64"
         BuilderContainer(DEFAULT_PYTHON_VERSION, DEFAULT_BUILD_TYPE, architecture).run()
     container = _get_container(
         team,
+        operating_system,
         workers,
         worker_id,
         parallelism_per_worker,
@@ -195,6 +207,7 @@ def _add_default_except_tags(except_tags: str) -> str:
 
 def _get_container(
     team: str,
+    operating_system: str,
     workers: int,
     worker_id: int,
     parallelism_per_worker: int,
@@ -208,15 +221,27 @@ def _get_container(
     shard_start = worker_id * parallelism_per_worker
     shard_end = (worker_id + 1) * parallelism_per_worker
 
-    return LinuxTesterContainer(
-        build_name or f"{team}build",
-        test_envs=test_env,
-        shard_count=shard_count,
-        shard_ids=list(range(shard_start, shard_end)),
-        gpus=gpus,
-        skip_ray_installation=skip_ray_installation,
-        build_type=build_type,
-    )
+    if operating_system == "linux":
+        return LinuxTesterContainer(
+            build_name or f"{team}build",
+            test_envs=test_env,
+            shard_count=shard_count,
+            shard_ids=list(range(shard_start, shard_end)),
+            gpus=gpus,
+            skip_ray_installation=skip_ray_installation,
+            build_type=build_type,
+        )
+
+    if operating_system == "windows":
+        return WindowsTesterContainer(
+            build_name or f"{team}build",
+            test_envs=test_env,
+            shard_count=shard_count,
+            shard_ids=list(range(shard_start, shard_end)),
+            skip_ray_installation=skip_ray_installation,
+        )
+
+    assert False, f"Unsupported operating system: {operating_system}"
 
 
 def _get_tag_matcher(tag: str) -> str:

cpp/src/ray/runtime/task/local_mode_task_submitter.cc

@@ -85,8 +85,13 @@ ObjectID LocalModeTaskSubmitter::Submit(InvocationSpec &invocation,
         TaskID::ForActorCreationTask(invocation.actor_id);
     const ObjectID actor_creation_dummy_object_id =
         ObjectID::FromIndex(actor_creation_task_id, 1);
-    builder.SetActorTaskSpec(
-        invocation.actor_id, actor_creation_dummy_object_id, invocation.actor_counter);
+    // NOTE: Ray CPP doesn't support retries and retry_exceptions.
+    builder.SetActorTaskSpec(invocation.actor_id,
+                             actor_creation_dummy_object_id,
+                             /*max_retries=*/0,
+                             /*retry_exceptions=*/false,
+                             /*serialized_retry_exception_allowlist=*/"",
+                             invocation.actor_counter);
   } else {
     throw RayException("unknown task type");
   }

cpp/src/ray/runtime/task/native_task_submitter.cc

@@ -69,10 +69,17 @@ ObjectID NativeTaskSubmitter::Submit(InvocationSpec &invocation,
   options.generator_backpressure_num_objects = -1;
   std::vector<rpc::ObjectReference> return_refs;
   if (invocation.task_type == TaskType::ACTOR_TASK) {
+    // NOTE: Ray CPP doesn't support per-method max_retries and retry_exceptions
+    const auto native_actor_handle = core_worker.GetActorHandle(invocation.actor_id);
+    int max_retries = native_actor_handle->MaxTaskRetries();
+
     auto status = core_worker.SubmitActorTask(invocation.actor_id,
                                               BuildRayFunction(invocation),
                                               invocation.args,
                                               options,
+                                              max_retries,
+                                              /*retry_exceptions=*/false,
+                                              /*serialized_retry_exception_allowlist=*/"",
                                               return_refs);
     if (!status.ok()) {
       return ObjectID::Nil();

dashboard/BUILD

@@ -21,7 +21,6 @@ py_test_run_all_subdirectory(
     include = ["**/test*.py"],
     exclude = [
         "client/node_modules/**",
-        "modules/test/**", 
         "modules/job/tests/test_cli_integration.py",
         "modules/job/tests/test_http_job_server.py",
         "modules/node/tests/test_node.py", 
@@ -87,18 +86,21 @@ py_test(
     size = "small",
     srcs = ["tests/test_state_head.py"],
     tags = ["team:core"],
+    deps = [":conftest"],
 )
 
 py_test(
     name = "test_serve_dashboard",
     size = "large",
     srcs = ["modules/serve/tests/test_serve_dashboard.py"],
     tags = ["team:serve"],
+    deps = [":conftest"],
 )
 
 py_test(
     name = "test_data_head",
     size = "small",
     srcs = ["modules/data/tests/test_data_head.py"],
     tags = ["team:data"],
+    deps = [":conftest"],
 )