CI: Build the Clang toolchain a single time for both onboard pipelines

As of the Clang 13 upgrade, we only need to build the toolchain once and can use that toolchain for both x86_64 and i686. To do this, this breaks the main Azure configuration into 3 "stages" (Lagom, Toolchain, and Serenity), where the Serenity stage depends on the Toolchain stage. This has the added benefit of uploading a new prebuilt toolchain cache sooner than before, which should help alleviate pressure from PRs.
Author: https://github.com/trflynn89 Commit: https://github.com/SerenityOS/serenity/commit/b11a34330a1 Pull-request: https://github.com/SerenityOS/serenity/pull/10516
2024-11-25 00:50:22 +00:00 · 2021-10-17 12:45:04 -04:00 · 2021-10-17 12:45:04 -04:00 · b11a34330a · 2024-07-18 02:11:43 +09:00
commit b11a34330a
parent 957f98805a
4 changed files with 95 additions and 51 deletions
--- a/Meta/Azure/Caches.yml
+++ b/Meta/Azure/Caches.yml
@ -4,8 +4,8 @@ parameters:
  toolchain: 'gcc'
  build_directory: ''
  ccache_version: 1 # Increment this number if CI has trouble with ccache.
-  serenity_ccache_path: $(CCACHE_DIR)
-  toolchain_ccache_path: $(CCACHE_DIR)
+  serenity_ccache_path: ''
+  toolchain_ccache_path: ''
  toolchain_ccache_size: $(CCACHE_MAXSIZE)
  with_unicode_caches: true

@ -18,7 +18,7 @@ steps:
    - ${{ if eq(parameters.toolchain, 'clang') }}:
      - task: Cache@2
        inputs:
-          key: '"toolchain" | "${{ parameters.arch }}" | Toolchain/BuildClang.sh | Toolchain/Patches/*[!gcc].patch | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
+          key: '"toolchain" | "x86_64" | Toolchain/BuildClang.sh | Toolchain/Patches/*[!gcc].patch | Userland/Libraries/LibC/**/*.h | Userland/Libraries/LibPthread/**/*.h'
          path: $(Build.SourcesDirectory)/Toolchain/Cache
        displayName: 'Toolchain Prebuilt Cache'
    - ${{ if eq(parameters.toolchain, 'gcc') }}:
@ -28,26 +28,33 @@ steps:
          path: $(Build.SourcesDirectory)/Toolchain/Cache
        displayName: 'Toolchain Prebuilt Cache'

+    - ${{ if ne(parameters.toolchain_ccache_path, '') }}:
+      - task: Cache@2
+        inputs:
+          key: '"toolchain ccache" | "x86_64" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
+          restoreKeys: |
+            "toolchain ccache" | "x86_64" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
+          path: ${{ parameters.toolchain_ccache_path }}
+        displayName: 'Toolchain Compiler Cache'
+
+      - script: |
+          CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -M ${{ parameters.toolchain_ccache_size }}
+          CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -s
+        displayName: 'Configure Toolchain ccache'
+
+  - ${{ if ne(parameters.serenity_ccache_path, '') }}:
    - task: Cache@2
      inputs:
-        key: '"toolchain ccache" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
+        key: '"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
        restoreKeys: |
-          "toolchain ccache" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
-        path: ${{ parameters.toolchain_ccache_path }}
-      displayName: 'Toolchain Compiler Cache'
+          "ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
+        path: ${{ parameters.serenity_ccache_path }}
+      displayName: 'Serenity Compiler Cache'

    - script: |
-        CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -M ${{ parameters.toolchain_ccache_size }}
-        CCACHE_DIR=${{ parameters.toolchain_ccache_path }} ccache -s
-      displayName: 'Configure Toolchain ccache'
-
-  - task: Cache@2
-    inputs:
-      key: '"ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}" | "$(timestamp)"'
-      restoreKeys: |
-        "ccache" | "${{ parameters.os }}" | "${{ parameters.arch }}" | "${{ parameters.toolchain }}" | "${{ parameters.ccache_version }}"
-      path: ${{ parameters.serenity_ccache_path }}
-    displayName: 'Serenity Compiler Cache'
+        CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -M 5G
+        CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -s
+      displayName: 'Configure Serenity ccache'

  - ${{ if eq(parameters.with_unicode_caches, true) }}:
    - task: Cache@2
@ -61,8 +68,3 @@ steps:
        key: '"unicode_locale" | Meta/CMake/unicode_data.cmake'
        path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/CLDR
      displayName: 'UnicodeLocale Cache'
-
-  - script: |
-      CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -M 5G
-      CCACHE_DIR=${{ parameters.serenity_ccache_path }} ccache -s
-    displayName: 'Configure Serenity ccache'
--- a/Meta/Azure/Serenity.yml
+++ b/Meta/Azure/Serenity.yml
@ -8,10 +8,6 @@ jobs:
    variables:
    - name: SERENITY_CCACHE_DIR
      value: $(Build.SourcesDirectory)/.ccache
-    - name: LLVM_CCACHE_DIR
-      value: $(Build.SourcesDirectory)/Toolchain/.ccache
-    - name: LLVM_CCACHE_MAXSIZE
-      value: 20GB

    pool:
      vmImage: ubuntu-20.04
@ -27,14 +23,11 @@ jobs:
        toolchain: 'clang'
        build_directory: 'Build/${{ parameters.arch }}clang'
        serenity_ccache_path: '$(SERENITY_CCACHE_DIR)'
-        toolchain_ccache_path: '$(LLVM_CCACHE_DIR)'
-        toolchain_ccache_size: '$(LLVM_CCACHE_MAXSIZE)'

-    - script: ./Toolchain/BuildClang.sh --ci
-      displayName: Build Toolchain
+    - script: ./Toolchain/BuildClang.sh
+      displayName: Extract Toolchain
      env:
        TRY_USE_LOCAL_TOOLCHAIN: 'y'
-        ARCH: '${{ parameters.arch }}'

    - script: |
        mkdir -p Build/superbuild
--- a/Meta/Azure/Toolchain.yml
+++ b/Meta/Azure/Toolchain.yml
@ -0,0 +1,34 @@
+jobs:
+  - job: 'Serenity_Clang_Toolchain'
+    timeoutInMinutes: 0 # Setting to 0 means the maximum allowed timeout is used.
+
+    variables:
+    - name: LLVM_CCACHE_DIR
+      value: $(Build.SourcesDirectory)/Toolchain/.ccache
+    - name: LLVM_CCACHE_MAXSIZE
+      value: 20GB
+
+    pool:
+      vmImage: ubuntu-20.04
+
+    steps:
+    - template: Setup.yml
+      parameters:
+        os: 'Serenity'
+
+    - template: Caches.yml
+      parameters:
+        toolchain: 'clang'
+        toolchain_ccache_path: '$(LLVM_CCACHE_DIR)'
+        toolchain_ccache_size: '$(LLVM_CCACHE_MAXSIZE)'
+        with_unicode_caches: false
+
+    - script: ./Toolchain/BuildClang.sh --ci
+      displayName: Build Toolchain
+      env:
+        TRY_USE_LOCAL_TOOLCHAIN: 'y'
+
+    - script: |
+        echo "##[section]Toolchain Cache"
+        CCACHE_DIR='$(LLVM_CCACHE_DIR)' ccache -s
+      displayName: 'Cache Stats'
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -4,26 +4,41 @@ trigger:
    include:
    - master

-jobs:
-  - template: Meta/Azure/Lagom.yml
-    parameters:
-      os: 'Linux'
-      lagom_lints: true
+stages:
+  - stage: Lagom
+    dependsOn: []

-  - template: Meta/Azure/Lagom.yml
-    parameters:
-      os: 'Linux'
-      fuzzer: 'Fuzz'
+    jobs:
+      - template: Meta/Azure/Lagom.yml
+        parameters:
+          os: 'Linux'
+          lagom_lints: true

-  - template: Meta/Azure/Lagom.yml
-    parameters:
-      os: 'macOS'
-      allow_test_failures: true
+      - template: Meta/Azure/Lagom.yml
+        parameters:
+          os: 'Linux'
+          fuzzer: 'Fuzz'

-  - template: Meta/Azure/Serenity.yml
-    parameters:
-      arch: 'i686'
+      - template: Meta/Azure/Lagom.yml
+        parameters:
+          os: 'macOS'
+          allow_test_failures: true

-  - template: Meta/Azure/Serenity.yml
-    parameters:
-      arch: 'x86_64'
+
+  - stage: Toolchain
+    dependsOn: []
+
+    jobs:
+      - template: Meta/Azure/Toolchain.yml
+
+  - stage: SerenityOS
+    dependsOn: Toolchain
+
+    jobs:
+      - template: Meta/Azure/Serenity.yml
+        parameters:
+          arch: 'i686'
+
+      - template: Meta/Azure/Serenity.yml
+        parameters:
+          arch: 'x86_64'