From 73b91d472b4c949029ca78f3ece1668a9c2f850d Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Sat, 20 Jun 2026 08:41:13 -0500 Subject: [PATCH 1/2] ci: cache sccache across remaining C++ build legs --- .github/workflows/aws_test.yml | 22 ++++++++- .github/workflows/sanitizer_test.yml | 24 +++++++++- .github/workflows/sql_catalog_test.yml | 23 +++++++++ .github/workflows/test.yml | 64 ++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 6 deletions(-) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 4d58edddb..19125e180 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -74,6 +74,8 @@ jobs: AWS_DEFAULT_REGION: us-east-1 AWS_ENDPOINT_URL: http://127.0.0.1:9000 AWS_EC2_METADATA_DISABLED: "TRUE" + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -113,11 +115,29 @@ jobs: if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_minio.sh + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} + restore-keys: | + sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}- + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build and test Iceberg shell: bash env: CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }} - run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }} + run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }} + - name: Show sccache stats + shell: bash + run: sccache --show-stats + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} # Exercise the Meson build with SigV4 enabled (resolves aws-cpp-sdk-core via # its CMake config, not pkg-config whose Cflags force -std=c++11). diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index f4edecfec..e461bd1ed 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -39,6 +39,9 @@ jobs: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} name: "ASAN and UBSAN Tests" runs-on: ubuntu-24.04 + env: + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -47,14 +50,33 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-sanitizer-ubuntu-${{ github.run_id }} + restore-keys: | + sccache-sanitizer-ubuntu- + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Configure and Build with ASAN & UBSAN env: CC: gcc-14 CXX: g++-14 run: | mkdir build && cd build - cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache cmake --build . --verbose + - name: Show sccache stats + shell: bash + run: sccache --show-stats + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-sanitizer-ubuntu-${{ github.run_id }} - name: Run Tests working-directory: build env: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index cc95169af..dc5c5911e 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -61,6 +61,9 @@ jobs: runs-on: windows-2025 cmake_build_type: Release cmake_extra_args: -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake + env: + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -85,6 +88,15 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-sqlcatalog-${{ matrix.runs-on }}-${{ github.run_id }} + restore-keys: | + sccache-sqlcatalog-${{ matrix.runs-on }}- + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Configure Iceberg shell: bash run: | @@ -96,10 +108,21 @@ jobs: -DICEBERG_BUILD_REST=OFF \ -DICEBERG_BUILD_SQL_CATALOG=ON \ -DICEBERG_SQL_SQLITE=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ ${{ matrix.cmake_extra_args }} - name: Build SQL catalog tests shell: bash run: cmake --build build --target sql_catalog_test + - name: Show sccache stats + shell: bash + run: sccache --show-stats + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-sqlcatalog-${{ matrix.runs-on }}-${{ github.run_id }} - name: Run SQL catalog tests shell: bash run: ctest --test-dir build -R '^sql_catalog_test$' --output-on-failure -C ${{ matrix.cmake_build_type }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 652999f61..6a2f4ca82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,9 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false + env: + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -53,12 +56,30 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-test-ubuntu-${{ github.run_id }} + restore-keys: | + sccache-test-ubuntu- + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build Iceberg shell: bash env: CC: gcc-14 CXX: g++-14 - run: ci/scripts/build_iceberg.sh $(pwd) ON + run: ci/scripts/build_iceberg.sh $(pwd) ON ON + - name: Show sccache stats + shell: bash + run: sccache --show-stats + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-test-ubuntu-${{ github.run_id }} - name: Build Example shell: bash env: @@ -72,14 +93,35 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false + env: + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-test-macos-${{ github.run_id }} + restore-keys: | + sccache-test-macos- + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build Iceberg shell: bash - run: ci/scripts/build_iceberg.sh $(pwd) + run: ci/scripts/build_iceberg.sh $(pwd) OFF ON + - name: Show sccache stats + shell: bash + run: sccache --show-stats + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-test-macos-${{ github.run_id }} - name: Build Example shell: bash run: ci/scripts/build_example.sh $(pwd)/example @@ -90,6 +132,9 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false + env: + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -103,17 +148,28 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-test-windows-${{ github.run_id }} + restore-keys: | + sccache-test-windows- - name: Setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build Iceberg shell: pwsh - env: - SCCACHE_GHA_ENABLED: "true" run: | $ErrorActionPreference = "Stop" bash -lc 'ci/scripts/build_iceberg.sh $(pwd) OFF ON' if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } sccache --show-stats + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-test-windows-${{ github.run_id }} - name: Build Example shell: pwsh run: | From 4a0fff5bb6587caac46cdf7229b71ffa2c115e88 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Sat, 20 Jun 2026 08:09:04 -0500 Subject: [PATCH 2/2] ci: cache sccache on Meson build legs --- .github/workflows/aws_test.yml | 25 +++++++++++++++++++++++-- .github/workflows/test.yml | 29 +++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 19125e180..388d7508e 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -146,6 +146,9 @@ jobs: name: Meson SigV4 (AMD64 Ubuntu 24.04) runs-on: ubuntu-24.04 timeout-minutes: 45 + env: + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" steps: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -181,9 +184,20 @@ jobs: echo "::error::vcpkg install failed after 3 attempts" exit 1 - name: Set Ubuntu Compilers + # Wrap the compiler with sccache: Meson uses an explicit CC/CXX verbatim, + # so the launcher must be prepended here to route compiles through sccache. run: | - echo "CC=gcc-14" >> $GITHUB_ENV - echo "CXX=g++-14" >> $GITHUB_ENV + echo "CC=sccache gcc-14" >> $GITHUB_ENV + echo "CXX=sccache g++-14" >> $GITHUB_ENV + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-meson-sigv4-${{ github.run_id }} + restore-keys: | + sccache-meson-sigv4- + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build and test Iceberg shell: bash env: @@ -191,4 +205,11 @@ jobs: run: | meson setup builddir -Dsigv4=enabled meson compile -C builddir + sccache --show-stats meson test -C builddir --timeout-multiplier 0 --print-errorlogs + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-meson-sigv4-${{ github.run_id }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a2f4ca82..cf35ac4d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -180,6 +180,9 @@ jobs: name: Meson - ${{ matrix.title }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 30 + env: + SCCACHE_DIR: ${{ github.workspace }}/.sccache + SCCACHE_CACHE_SIZE: "2G" strategy: max-parallel: 15 fail-fast: false @@ -208,24 +211,34 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt - name: Set Ubuntu Compilers + # Wrap the compiler with sccache: Meson auto-detects sccache for the + # default compiler (macOS/Windows), but uses an explicit CC/CXX verbatim, + # so the launcher must be prepended here to route Ubuntu compiles through it. if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} run: | - echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV - echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV + echo "CC=sccache ${{ matrix.CC }}" >> $GITHUB_ENV + echo "CXX=sccache ${{ matrix.CXX }}" >> $GITHUB_ENV + - name: Restore sccache cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} + restore-keys: | + sccache-meson-${{ matrix.runs-on }}- - name: Setup sccache - if: ${{ startsWith(matrix.runs-on, 'windows') }} uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - name: Enable sccache - if: ${{ startsWith(matrix.runs-on, 'windows') }} - shell: bash - run: echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" - name: Build Iceberg run: | meson setup builddir ${{ matrix.meson-setup-args || '' }} meson compile -C builddir - name: Show sccache stats - if: ${{ startsWith(matrix.runs-on, 'windows') }} run: sccache --show-stats + - name: Save sccache cache + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ${{ github.workspace }}/.sccache + key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} - name: Test Iceberg run: | meson test -C builddir --timeout-multiplier 0 --print-errorlogs