Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 56 additions & 86 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,96 +1,57 @@
name: GitHub-hosted
name: CI

on:
push:
branches: [master, main]
pull_request:

env:
CMAKE_BUILD_PARALLEL_LEVEL: 4
permissions:
contents: read

concurrency:
group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
# PR-only: don't cancel an in-flight push validation on master.
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
build-and-test:
build-test:
name: ${{ matrix.os }}/llvm${{ matrix.llvm }}/py${{ matrix.py }}/c++${{ matrix.cxx }}${{ matrix.vg && '/vg' || '' }}${{ matrix.cling && '/cling' || '' }}
strategy:
fail-fast: false
# Mirrors CppInterOp's cppyy PR cells: clang-repl on the same OS/arch/LLVM
# /valgrind combos at Python 3.14, plus a cling cell. Breadth (3.12/3.13,
# C++17, LLVM 20, arm) lives in nightly.
matrix:
include:
- os: ubuntu-24.04
llvm: 20
python: "3.12"
runtime_cxx_standard: 17
- os: ubuntu-24.04
llvm: 21
python: "3.13"
runtime_cxx_standard: 20
- os: ubuntu-24.04
llvm: 21
python: "3.14"
runtime_cxx_standard: 20
- os: macos-15
llvm: 20
python: "3.12"
runtime_cxx_standard: 20

runs-on: ${{ matrix.os }}
name: ${{ matrix.os }} / LLVM ${{ matrix.llvm }} / Py ${{ matrix.python }} / C++${{ matrix.runtime_cxx_standard }}

steps:
- uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}

- name: Install dependencies (Linux)
if: runner.os == 'Linux'
run: |
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-${{ matrix.llvm }} main" \
| sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update
sudo apt-get install -y \
llvm-${{ matrix.llvm }}-dev \
libclang-${{ matrix.llvm }}-dev \
clang-${{ matrix.llvm }} \
libpolly-${{ matrix.llvm }}-dev \
libboost-dev libeigen3-dev

- name: Install dependencies (macOS)
if: runner.os == 'macOS'
run: brew install llvm@${{ matrix.llvm }} boost eigen

- name: pip install CppJIT
run: |
pip install pytest numpy psutil
pip install numba || true
pip install . -v --config-settings=cmake.define.LLVM_DIR=${{
runner.os == 'macOS'
&& format('/opt/homebrew/opt/llvm@{0}/lib/cmake/llvm', matrix.llvm)
|| format('/usr/lib/llvm-{0}/lib/cmake/llvm', matrix.llvm)
}}

- name: Build test dictionaries
run: make -j4
working-directory: test

- name: Run tests
run: python -m pytest -ra --tb=short -q 2>&1 | tee ../test-output.txt
working-directory: test
env:
CPPINTEROP_EXTRA_INTERPRETER_ARGS: "-std=c++${{ matrix.runtime_cxx_standard }}"

- name: Upload result
if: always()
uses: actions/upload-artifact@v4
with:
name: result-${{ matrix.os }}-llvm${{ matrix.llvm }}-py${{ matrix.python }}-cpp${{ matrix.runtime_cxx_standard }}
path: test-output.txt
- { os: ubuntu-24.04, llvm: '21', flavor: system, py: '3.14', cxx: '20', vg: true }

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We need name: here because it duplicates the strings and in the github ci bar it shows twice.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

vg:false should be default and omitted.

- { os: ubuntu-24.04, llvm: '22', flavor: '', py: '3.14', cxx: '20', vg: false }
- { os: macos-26, llvm: '21', flavor: system, py: '3.14', cxx: '20', vg: false }
- { os: macos-26-intel, llvm: '21', flavor: system, py: '3.14', cxx: '20', vg: false }
# cling backend, against the cached llvm-root (cling-llvm20) recipe cell.
- { os: ubuntu-24.04, llvm: '20', flavor: cling, fv: cling-llvm20, cling: true, py: '3.14', cxx: '20', vg: false }

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is fv, isn't cling:true redundant?

uses: aaronj0/ci-workflows/.github/workflows/cppjit.yml@cppjit-reusable
# NOTE: retarget to compiler-research/ci-workflows/...@main once upstreamed.
with:
# CppInterOp is pinned (reusable workflow default) to a commit compatible
# with the current forks. Bump that pin after re-syncing the forks.
cppjit-repo: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
cppjit-ref: ${{ github.event.pull_request.head.sha || github.sha }}
os: ${{ matrix.os }}
llvm-version: ${{ matrix.llvm }}
llvm-flavor: ${{ matrix.flavor }}
llvm-flavor-version: ${{ matrix.fv || '' }}
python-version: ${{ matrix.py }}
cxx-standard: ${{ matrix.cxx }}
valgrind: ${{ matrix.vg }}
run-sweeps: true
cling: ${{ matrix.cling || false }}

report:
if: always() && github.event_name == 'pull_request'
needs: build-and-test
if: ${{ always() && github.event_name == 'pull_request' }}
needs: build-test
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/download-artifact@v4
Expand All @@ -105,20 +66,29 @@ jobs:

const rows = fs.readdirSync('.').filter(d => d.startsWith('result-')).sort().map(d => {
const cfg = d.replace('result-', '');
const file = `${d}/test-output.txt`;
const file = `${d}/pytest-summary.txt`;
const lines = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n') : [];
const result = lines.length ? lines.at(-1) : 'no output';
return `| ${cfg} | \`${result}\` |`;
});

const body = [marker, '## Test Results', '| Configuration | Result |', '|---|---|', ...rows].join('\n');
const { data: comments } = await github.rest.issues.listComments({
...context.repo, issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.startsWith(marker));

if (existing) {
await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body });
} else {
await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body });
// Run summary works even with a read-only token (fork PRs).
await core.summary.addRaw(body).write();

// PR comment needs pull-requests:write, downgraded to read-only on
// fork PRs — don't fail the job when the write errors out.
try {
const { data: comments } = await github.rest.issues.listComments({
...context.repo, issue_number: context.issue.number,
});
const existing = comments.find(c => c.body.startsWith(marker));
if (existing) {
await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body });
} else {
await github.rest.issues.createComment({ ...context.repo, issue_number: context.issue.number, body });
}
} catch (e) {
core.warning(`Could not post PR comment (likely a fork PR with a read-only token): ${e.message}`);
}
92 changes: 92 additions & 0 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
name: Nightly

# Exhaustive run on a schedule: the full matrix with the xfail sweeps and
# valgrind on every Linux cell, against the pinned CppInterOp (reusable
# default). On any failure a tracking issue is opened/updated. To restore
# upstream-drift detection, add a cell with `cppinterop-ref: main` (will be
# red until the forks are re-synced to CppInterOp's strong-types API).

on:
schedule:
- cron: '30 2 * * *'
workflow_dispatch:

permissions:
contents: read

concurrency:
group: nightly-${{ github.workflow }}
cancel-in-progress: true

jobs:
build-test:
name: ${{ matrix.os }}/llvm${{ matrix.llvm }}/py${{ matrix.py }}/c++${{ matrix.cxx }}${{ matrix.vg && '/vg' || '' }}${{ matrix.cling && '/cling' || '' }}
strategy:
fail-fast: false
matrix:
# Superset of the PR matrix + breadth (py 3.12/3.13, C++17, LLVM 20, arm)
# and one cling cell. valgrind only on LLVM 21 cells: vendored etc/ has
# clang21 supps only, so other majors would surface unsuppressed noise.
include:
- { os: ubuntu-24.04, llvm: '20', flavor: system, py: '3.12', cxx: '17', vg: false }
- { os: ubuntu-24.04, llvm: '21', flavor: system, py: '3.13', cxx: '20', vg: true }
- { os: ubuntu-24.04, llvm: '21', flavor: system, py: '3.14', cxx: '20', vg: true }
- { os: ubuntu-24.04, llvm: '22', flavor: '', py: '3.14', cxx: '20', vg: false }
- { os: ubuntu-24.04, llvm: '21', flavor: system, py: '3.12', cxx: '20', vg: true }
- { os: ubuntu-24.04-arm, llvm: '21', flavor: system, py: '3.13', cxx: '20', vg: true }
- { os: ubuntu-24.04-arm, llvm: '20', flavor: system, py: '3.12', cxx: '17', vg: false }
- { os: macos-26, llvm: '21', flavor: system, py: '3.12', cxx: '20', vg: false }
- { os: macos-26-intel, llvm: '21', flavor: system, py: '3.12', cxx: '20', vg: false }
# cling backend, against the cached llvm-root (cling-llvm20) recipe cell.
- { os: ubuntu-24.04, llvm: '20', flavor: cling, fv: cling-llvm20, cling: true, py: '3.13', cxx: '20', vg: false }
uses: aaronj0/ci-workflows/.github/workflows/cppjit.yml@cppjit-reusable
# NOTE: retarget to compiler-research/ci-workflows/...@main once upstreamed.
with:
os: ${{ matrix.os }}
llvm-version: ${{ matrix.llvm }}
llvm-flavor: ${{ matrix.flavor }}
llvm-flavor-version: ${{ matrix.fv || '' }}
python-version: ${{ matrix.py }}
cxx-standard: ${{ matrix.cxx }}
valgrind: ${{ matrix.vg }}
run-sweeps: true

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This name makes no sense -- maybe run-all-crashing-tests or something is better.

cling: ${{ matrix.cling || false }}

drift-issue:
needs: build-test
if: ${{ always() && needs.build-test.result == 'failure' }}
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- uses: actions/github-script@v7
with:
script: |
const marker = '<!-- cppjit-nightly-drift -->';
const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const body = [
marker,
'## Nightly build/test failed',
'',
`The nightly run (CppJIT \`master\` against CppInterOp \`main\`) failed.`,
`Most likely an upstream CppInterOp drift; investigate and sync if needed.`,
'',
`Run: ${run}`,
`Date: ${new Date().toISOString()}`,
].join('\n');

const existing = await github.paginate(github.rest.issues.listForRepo, {
...context.repo, state: 'open', labels: 'nightly-drift',
});
const hit = existing.find(i => (i.body || '').startsWith(marker));
if (hit) {
await github.rest.issues.createComment({ ...context.repo, issue_number: hit.number, body });
} else {
await github.rest.issues.create({
...context.repo,
title: 'Nightly: CppJIT broken against CppInterOp main',
labels: ['nightly-drift'],
body,
});
}
41 changes: 5 additions & 36 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ include(GNUInstallDirs)
# This option won't make a lot of sense since we only ship the shared library in site-packages
# Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests?
option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF)
set(CPPINTEROP_GIT_TAG "main" CACHE STRING "")
set(CPPINTEROP_GIT_TAG "3f5684b1127a84c6bcbbdc0750533936458634d6" CACHE STRING "")

set(Python_FIND_VIRTUALENV ONLY)
find_package(Python COMPONENTS Interpreter Development)
Expand Down Expand Up @@ -78,41 +78,10 @@ else()
set(CPPINTEROP_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/cppjit/cppyy_backend")
endif()

set(_interop_cmake_args
-DLLVM_DIR=${LLVM_DIR}
-DCPPINTEROP_USE_REPL=ON
-DCPPINTEROP_USE_CLING=OFF
-DCPPINTEROP_ENABLE_TESTING=${CPPJIT_ENABLE_CPPINTEROP_TESTS}
-DBUILD_SHARED_LIBS=ON
-DCMAKE_INSTALL_PREFIX=${CPPINTEROP_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_STANDARD=17
)
if(Clang_DIR)
list(APPEND _interop_cmake_args -DClang_DIR=${Clang_DIR})
endif()
if(CMAKE_C_COMPILER)
list(APPEND _interop_cmake_args -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
endif()
if(CMAKE_CXX_COMPILER)
list(APPEND _interop_cmake_args -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
endif()

ExternalProject_Add(CppInterOp
GIT_REPOSITORY https://github.com/compiler-research/CppInterOp.git
GIT_TAG ${CPPINTEROP_GIT_TAG}
GIT_SHALLOW ON
PREFIX "${CMAKE_BINARY_DIR}/CppInterOp"
CMAKE_ARGS ${_interop_cmake_args}
BUILD_BYPRODUCTS
"${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}"
LOG_DOWNLOAD ON
LOG_CONFIGURE ON
LOG_BUILD ON
LOG_INSTALL ON
LOG_OUTPUT_ON_FAILURE ON
)
# Backend selection (repl default; cling for ROOT) and CppInterOp ExternalProject
# build configuration in this module, defined in cppjit_add_cppinterop().
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/CppInterOpExternal.cmake)
cppjit_add_cppinterop()

# this libcppyy.so merges both CPyCppyy and clingwrapper
file(GLOB CPYCPPYY_SOURCES CONFIGURE_DEPENDS src/CPyCppyy/src/*.cxx)
Expand Down
76 changes: 76 additions & 0 deletions cmake/CppInterOpExternal.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Configures the CppInterOp ExternalProject. Default backend is clang-repl;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need this file? If it has something to do with the ci, then then it defeats the purpose of the ci-workflows changes.

# CPPJIT_USE_CLING builds CppInterOp against a prebuilt cling instead (for
# ROOT, cppjit does NOT build cling itself, it accepts a build).
include_guard(GLOBAL)
include(ExternalProject)

# Developer toggle: Cling from either ROOT or standalone supplies LLVM_DIR/Clang_DIR
option(CPPJIT_USE_CLING "Build CppInterOp against a prebuilt Cling C++ Interpreter (ROOT)" OFF)
mark_as_advanced(CPPJIT_USE_CLING)

set(Cling_DIR "" CACHE PATH "Cling CMake config dir; derived from LLVM_DIR when empty (cling mode only)")
# Surface Cling_DIR in ccmake/GUI only when cling mode is on.
if(CPPJIT_USE_CLING)
mark_as_advanced(CLEAR Cling_DIR)
else()
mark_as_advanced(Cling_DIR)
endif()

function(cppjit_add_cppinterop)
set(_args
-DLLVM_DIR=${LLVM_DIR}
-DCPPINTEROP_ENABLE_TESTING=${CPPJIT_ENABLE_CPPINTEROP_TESTS}
-DBUILD_SHARED_LIBS=ON
-DCMAKE_INSTALL_PREFIX=${CPPINTEROP_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_STANDARD=17
)

if(CPPJIT_USE_CLING)
set(_cling_dir "${Cling_DIR}")
if(NOT _cling_dir)
# LLVM_DIR is <prefix>/lib/cmake/llvm; cling config sits at the
# sibling <prefix>/lib/cmake/cling (setup-llvm flavor:cling layout).
get_filename_component(_prefix "${LLVM_DIR}" DIRECTORY) # <prefix>/lib/cmake
get_filename_component(_prefix "${_prefix}" DIRECTORY) # <prefix>/lib
get_filename_component(_prefix "${_prefix}" DIRECTORY) # <prefix>
set(_cling_dir "${_prefix}/lib/cmake/cling")
endif()
message(STATUS "CppInterOp backend: cling (Cling_DIR=${_cling_dir})")
list(APPEND _args
-DCPPINTEROP_USE_CLING=ON
-DCPPINTEROP_USE_REPL=OFF
-DCling_DIR=${_cling_dir}
)
else()
list(APPEND _args
-DCPPINTEROP_USE_REPL=ON
-DCPPINTEROP_USE_CLING=OFF
)
endif()

if(Clang_DIR)
list(APPEND _args -DClang_DIR=${Clang_DIR})
endif()
if(CMAKE_C_COMPILER)
list(APPEND _args -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
endif()
if(CMAKE_CXX_COMPILER)
list(APPEND _args -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
endif()

ExternalProject_Add(CppInterOp
GIT_REPOSITORY https://github.com/compiler-research/CppInterOp.git
GIT_TAG ${CPPINTEROP_GIT_TAG}
PREFIX "${CMAKE_BINARY_DIR}/CppInterOp"
CMAKE_ARGS ${_args}
BUILD_BYPRODUCTS
"${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}"
LOG_DOWNLOAD ON
LOG_CONFIGURE ON
LOG_BUILD ON
LOG_INSTALL ON
LOG_OUTPUT_ON_FAILURE ON
)
endfunction()
Loading
Loading