ci: use HF cache

Lightning-AI · Mar 11, 2025 · 7d426b5 · 7d426b5
1 parent 567e256
commit 7d426b5
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 4 deletions.
diff --git a/.azure/gpu-test-with-thunder.yml b/.azure/gpu-test-with-thunder.yml
@@ -19,10 +19,13 @@ jobs:
     pool: "lit-rtx-3090"
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
+      TRANSFORMERS_CACHE: "/var/tmp/hf/transformers"
+      HF_HOME: "/var/tmp/hf/home"
+      HF_HUB_CACHE: "/var/tmp/hf/hub"
       CI: "true"
     container:
       image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
-      options: "--gpus=all --shm-size=8gb"
+      options: "--gpus=all --shm-size=8gb -v /var/tmp:/var/tmp"
     workspace:
       clean: all
     steps:

diff --git a/.azure/gpu-test.yml b/.azure/gpu-test.yml
@@ -13,16 +13,19 @@ pr:
       - "wip"
 
 jobs:
-  - job: testing
+  - job: base-testing
     timeoutInMinutes: "30"
     cancelTimeoutInMinutes: "2"
     pool: "lit-rtx-3090"
     variables:
       DEVICES: $( python -c 'print("$(Agent.Name)".split("_")[-1])' )
+      TRANSFORMERS_CACHE: "/var/tmp/hf/transformers"
+      HF_HOME: "/var/tmp/hf/home"
+      HF_HUB_CACHE: "/var/tmp/hf/hub"
       CI: "true"
     container:
       image: "pytorchlightning/pytorch_lightning:base-cuda-py3.10-torch2.2-cuda12.1.0"
-      options: "--gpus=all --shm-size=8gb"
+      options: "--gpus=all --shm-size=8gb -v /var/tmp:/var/tmp"
     workspace:
       clean: all
     steps:

diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
@@ -15,7 +15,10 @@ defaults:
     shell: bash
 
 env:
-  HF_TOKEN: ${{ secrets.HF_TOKEN }}
+  HF_HOME: ~/.cache/huggingface  # Define HF_HOME for caching
+  TRANSFORMERS_CACHE: ~/.cache/huggingface/transformers
+  DATASETS_CACHE: ~/.cache/huggingface/datasets
+  HF_DATASETS_CACHE: ~/.cache/huggingface/datasets
 
 jobs:
   testing-imports: