diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 00b8ff645ba..b634e72e511 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -71,8 +71,8 @@ jobs:
           # *and* as actual output variable, in this order.
           message("  set-output name=time::${current_date}")
           message("::set-output name=time::${current_date}")
-          message("  set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/BuildIt.sh') }}")
-          message("::set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*.patch', 'Toolchain/BuildIt.sh') }}")
+          message("  set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*[!llvm].patch', 'Toolchain/BuildIt.sh') }}")
+          message("::set-output name=libc_headers::${{ hashFiles('Userland/Libraries/LibC/**/*.h', 'Userland/Libraries/LibPthread/**/*.h', 'Toolchain/Patches/*[!llvm].patch', 'Toolchain/BuildIt.sh') }}")
 
       - name: Toolchain cache
         # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
diff --git a/Meta/Azure/Caches.yml b/Meta/Azure/Caches.yml
index d96aa4361aa..5ecfe58d687 100644
--- a/Meta/Azure/Caches.yml
+++ b/Meta/Azure/Caches.yml
@@ -7,16 +7,16 @@ parameters:
 
 steps:
   - ${{ if ne(parameters.arch, 'Lagom') }}:
-    - ${{ if eq(parameters.toolchain, 'gcc') }}:
-      - task: Cache@2
-        inputs:
-          key: '"toolchain" | "${{ parameters.arch }}" | Toolchain/BuildClang.sh | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
-          path: $(Build.SourcesDirectory)/Toolchain/Cache
-        displayName: 'Toolchain Cache'
     - ${{ if eq(parameters.toolchain, 'clang') }}:
       - task: Cache@2
         inputs:
-          key: '"toolchain" | "${{ parameters.arch }}" | Toolchain/BuildIt.sh | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
+          key: '"toolchain" | "${{ parameters.arch }}" | Toolchain/BuildClang.sh | Toolchain/Patches/*[!gcc].patch | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
+          path: $(Build.SourcesDirectory)/Toolchain/Cache
+        displayName: 'Toolchain Cache'
+    - ${{ if eq(parameters.toolchain, 'gcc') }}:
+      - task: Cache@2
+        inputs:
+          key: '"toolchain" | "${{ parameters.arch }}" | Toolchain/BuildIt.sh | Toolchain/Patches/*[!llvm].patch | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
           path: $(Build.SourcesDirectory)/Toolchain/Cache
         displayName: 'Toolchain Cache'