Skip to content

Commit 9ce2eab

Browse files
tpatkiggeorgakoudisdaboehmeslabasankoparasy
authored
Caliper integration with new PFA (Clang 18) (#190)
* Start with moving over docs, the easiest part... * Update github-actions to include Caliper * Add Caliper build option, FindCaliper.cmake, and update Setup3rdparty.cmake * update cmake setup_targets and setup_deps * Update to src/c/runtime/CMakeLists.txt * Add Caliper linking to src/c/test/CMakeLists.txt * Add deps and targers to src/c/config/ * Add caliper instrumentation functionality to perfflow_weave_common. Untested * Fix CMake typo in c/test/ * Attempt to address some Clang18 build errors * Code formatting * Attempt to fix InstIterator error * Format rst file * Remove debug statement from Setup3rdParty.cmake * Remove Adiak from this build. To be added with Spencer's build * Fix the build outpur msg for FOUND Caliper * Leave PERFFLOWASPECT_WITH_CALIPER OFF by default * Add a comment in the host config files about Caliper options * Caliper + PFA Python integration, untested * Update github actions to pick up Caliper python bindings * Update github actions to include pybind11 install * Debug msg * Debug msg * Tested and works, remove debug msgs. * Fix sharness test error caused due to PYTHONPATH updation * Try a conditional import of the pycaliper module, to fix the CI error * Try a Conditional import for pycaliper * Still attempting a conditional import * Update PYTHONPATH again * Another attempt at a conditional import * Remove debug msgs * Formatted * Clean up commits, add docs * Formatted Co-authored-by: Giorgis Georgakoudis <[email protected]> Co-authored-by: daboehme <[email protected]> Co-authored-by: Stephanie Brink <[email protected]> Co-authored-by: koparasy <[email protected]>
1 parent 42e81a5 commit 9ce2eab

18 files changed

Lines changed: 382 additions & 9 deletions

‎.github/workflows/github-actions.yml‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,49 +9,76 @@ jobs:
99
runs-on: ubuntu-24.04
1010
strategy:
1111
matrix:
12-
config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only]
12+
config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only, boilerplate_caliper]
1313

1414
include:
1515
- config: boilerplate
1616
PERFFLOWASPECT_WITH_CUDA: OFF
1717
PERFFLOWASPECT_WITH_MPI: ON
1818
PERFFLOWASPECT_WITH_MULTITHREADS: ON
1919
CMAKE_BUILD_TYPE: Debug
20+
PERFFLOWASPECT_WITH_CALIPER: OFF
2021

2122
- config: release
2223
PERFFLOWASPECT_WITH_CUDA: OFF
2324
PERFFLOWASPECT_WITH_MPI: ON
2425
PERFFLOWASPECT_WITH_MULTITHREADS: ON
2526
CMAKE_BUILD_TYPE: Release
27+
PERFFLOWASPECT_WITH_CALIPER: OFF
2628

2729
- config: debug_cuda_only
2830
PERFFLOWASPECT_WITH_CUDA: OFF
2931
PERFFLOWASPECT_WITH_MPI: OFF
3032
PERFFLOWASPECT_WITH_MULTITHREADS: OFF
3133
CMAKE_BUILD_TYPE: Debug
34+
PERFFLOWASPECT_WITH_CALIPER: OFF
3235

3336
- config: debug_mpi_only
3437
PERFFLOWASPECT_WITH_CUDA: OFF
3538
PERFFLOWASPECT_WITH_MPI: ON
3639
PERFFLOWASPECT_WITH_MULTITHREADS: OFF
3740
CMAKE_BUILD_TYPE: Debug
41+
PERFFLOWASPECT_WITH_CALIPER: OFF
3842

3943
- config: debug_threads_only
4044
PERFFLOWASPECT_WITH_CUDA: OFF
4145
PERFFLOWASPECT_WITH_MPI: OFF
4246
PERFFLOWASPECT_WITH_MULTITHREADS: ON
4347
CMAKE_BUILD_TYPE: Debug
48+
PERFFLOWASPECT_WITH_CALIPER: OFF
4449

50+
- config: boilerplate_caliper
51+
PERFFLOWASPECT_WITH_CUDA: OFF
52+
PERFFLOWASPECT_WITH_MPI: OFF
53+
PERFFLOWASPECT_WITH_MULTITHREADS: OFF
54+
CMAKE_BUILD_TYPE: Debug
55+
PERFFLOWASPECT_WITH_CALIPER: ON
56+
4557
steps:
4658
# Checkout PerfFlowAspect repository under $GITHUB_WORKSPACE
4759
- uses: actions/checkout@v2
4860

4961
- name: Setup Build Env
5062
run: |
5163
sudo apt-get update
52-
sudo apt install clang-18 llvm-dev libjansson-dev libssl-dev bison flex make cmake mpich
64+
sudo apt install clang-18 llvm-dev libjansson-dev libssl-dev bison flex make cmake mpich pybind11-dev
5365
clang++ --version
5466
67+
- name: Clone Caliper
68+
uses: actions/checkout@v2
69+
with:
70+
repository: LLNL/Caliper
71+
path: Caliper
72+
73+
- name: Build Caliper
74+
working-directory: Caliper
75+
run: |
76+
mkdir build && mkdir install
77+
cd build
78+
cmake -DWITH_PYTHON_BINDINGS=On -DCMAKE_INSTALL_PREFIX=../install ../
79+
make VERBOSE=1
80+
make install
81+
5582
- name: Compile check
5683
run: |
5784
cd src/c
@@ -62,6 +89,7 @@ jobs:
6289
export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CUDA=${{matrix.PERFFLOWASPECT_WITH_CUDA}}"
6390
export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MPI=${{matrix.PERFFLOWASPECT_WITH_MPI}}"
6491
export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MULTITHREADS=${{matrix.PERFFLOWASPECT_WITH_MULTITHREADS}}"
92+
export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CALIPER=${{matrix.PERFFLOWASPECT_WITH_CALIPER}} -Dcaliper_DIR=/home/runner/work/PerfFlowAspect/PerfFlowAspect/Caliper/install"
6593
echo -e ${CMAKE_OPTS}
6694
cmake ${CMAKE_OPTS} ..
6795
# build
@@ -97,6 +125,8 @@ jobs:
97125
cd src/python
98126
python -m pip install -r requirements.txt
99127
python -m pip list
128+
# Export path for caliper
129+
export PYTHONPATH="/home/runner/work/PerfFlowAspect/PerfFlowAspect/Caliper/install/lib/${{ matrix.python-version }}/site-packages:$PYTHONPATH"
100130
101131
- name: Run Python Smoke Tests
102132
run: |

‎docs/BasicTutorial.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ variable. Separate multiple variables with a colon as follows:
103103
+----------------------+--------------------------------------------------------------+---------------+---------------------------------+
104104
| log-format | Dump JSON events in array or object format | Array | Array, Object |
105105
+----------------------+--------------------------------------------------------------+---------------+---------------------------------+
106+
| caliper-enable | Toggle Caliper data collection on/off (for Python only). | False | True, False |
107+
+----------------------+--------------------------------------------------------------+---------------+---------------------------------+
106108

107109
**********************************************
108110
Visualization of PerfFlowAspect Output Files

‎docs/CaliperIntegration.rst‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
..
2+
# Copyright 2021 Lawrence Livermore National Security, LLC and other
3+
# PerfFlowAspect Project Developers. See the top-level LICENSE file for
4+
# details.
5+
#
6+
# SPDX-License-Identifier: LGPL-3.0
7+
8+
#####################
9+
Caliper Integration
10+
#####################
11+
12+
PerfFlowAspect can be built with Caliper to leverage collection of additional
13+
performance data, such as hardware performance counters on CPUs and GPU
14+
measurements on NVIDIA GPUs. `Caliper <https://github.com/llnl/caliper>`_ is an
15+
instrumentation and performance annotation library.
16+
17+
PerfFlowAspect supports the Caliper integration for both C/C++ and Python
18+
codebases. This enables a single PerfFlowAspect annotation to generate both
19+
`.pfw` trace files and `.cali` files, so users do not need any additional
20+
annotations to obtain Caliper data.
21+
22+
A Caliper install is required before building PerfFlowAspect in this case.
23+
Caliper can be configured at runtime, as shown in examples below.
24+
25+
*************
26+
C/C++ Build
27+
*************
28+
29+
Caliper needs to be is built with the same `clang` compiler chain as
30+
PerfFlowAspect. These can be specified with the `-DCMAKE_C_COMPILER` and
31+
`-DCMAKE_CXX_COMPILER`.
32+
33+
.. code:: bash
34+
35+
cmake -DCMAKE_C_COMPILER=<path-to-clang18-compiler> \
36+
-DCMAKE_C_COMPILER=<path-to-clang18++-compiler> \
37+
-DWITH_PYTHON_BINDINGS=On \
38+
-Dcaliper_DIR=<path-to-install>/share/lib/caliper ../
39+
40+
***************
41+
C/C++ Example
42+
***************
43+
44+
.. code:: bash
45+
46+
CALI_CONFIG=runtime-report ./smoketest
47+
CALI_CONFIG=runtime-report,output=test.cali ./smoketest
48+
49+
.. code:: bash
50+
51+
Path Min time/rank Max time/rank Avg time/rank Time %
52+
_Z3fooRKSs 0.004527 0.004527 0.004527 45.778137
53+
_Z3barv 0.004511 0.004511 0.004511 45.616341
54+
_Z3basv 0.000079 0.000079 0.000079 0.798867
55+
56+
.. code:: bash
57+
58+
CALI_PAPI_COUNTERS=PAPI_TOT_CYC,PAPI_L2_DCM CALI_SERVICES_ENABLE=event,trace,papi,report ./smoketest
59+
60+
.. code:: bash
61+
62+
event.begin#annotation papi.PAPI_TOT_CYC papi.PAPI_L2_DCM annotation region.count event.end#annotation
63+
_Z3fooRKSs 118289 679
64+
_Z3barv 200050 765 _Z3fooRKSs
65+
_Z3basv 115098 352 _Z3fooRKSs/_Z3barv
66+
66564 242 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv
67+
117061 385 _Z3fooRKSs/_Z3barv 1 _Z3barv
68+
93592 206 _Z3fooRKSs 1 _Z3fooRKSs
69+
_Z3fooRKSs 146308 332
70+
_Z3barv 87811 255 _Z3fooRKSs
71+
_Z3basv 84904 244 _Z3fooRKSs/_Z3barv
72+
34547 66 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv
73+
82540 168 _Z3fooRKSs/_Z3barv 1 _Z3barv
74+
80711 144 _Z3fooRKSs 1 _Z3fooRKSs
75+
_Z3fooRKSs 127765 183
76+
_Z3barv 85440 241 _Z3fooRKSs
77+
_Z3basv 82100 250 _Z3fooRKSs/_Z3barv
78+
33969 67 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv
79+
81511 161 _Z3fooRKSs/_Z3barv 1 _Z3barv
80+
77498 128 _Z3fooRKSs 1 _Z3fooRKSs
81+
_Z3fooRKSs 119853 164
82+
_Z3barv 83285 227 _Z3fooRKSs
83+
_Z3basv 82702 297 _Z3fooRKSs/_Z3barv
84+
34170 78 _Z3fooRKSs/_Z3barv/_Z3basv 1 _Z3basv
85+
81589 149 _Z3fooRKSs/_Z3barv 1 _Z3barv
86+
78920 119 _Z3fooRKSs 1 _Z3fooRKSs
87+
88+
.. code:: bash
89+
90+
PERFFLOW_OPTIONS='cpu-mem-usage=True:log-event=compact' CALI_CONFIG="load(time_exclusive.json),spot" ./smoketest
91+
92+
.. code:: bash
93+
94+
Path Min time/rank Max time/rank Avg time/rank Total time spot.channel
95+
_Z3fooRKSs 0.018068 0.018068 0.018068 0.018068 regionprofile
96+
_Z3barv 0.009124 0.009124 0.009124 0.009124 regionprofile
97+
_Z3basv 0.000074 0.000074 0.000074 0.000074 regionprofile
98+
99+
****************
100+
Python Example
101+
****************
102+
103+
For Python applications, Caliper can be enabled by setting the `caliper-enable`
104+
option. In this case, `PYTHONPATH` needs to be set to include the `pycaliper`
105+
package from Caliper installation.
106+
107+
.. code:: bash
108+
109+
export PYTHONPATH=<path-to-caliper-install>/lib/python<X.Y>/site-packages
110+
PERFFLOW_OPTIONS="caliper-enable=True" CALI_CONFIG=runtime-report ./smoketest.py
111+
112+
Inside main
113+
foo
114+
bar
115+
bas
116+
foo
117+
bar
118+
bas
119+
foo
120+
bar
121+
bas
122+
foo
123+
bar
124+
bas
125+
Path Time (E) Time (I) Time % (E) Time % (I)
126+
<function foo at 0x155546a24540> 0.004692 0.009423 47.547246 95.486397
127+
<function bar at 0x155546a24400> 0.004659 0.004731 47.213609 47.939151
128+
<function bas at 0x15554732de40> 0.000072 0.000072 0.725542 0.725542

‎docs/index.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled.
8585

8686
BuildingPerfFlowAspect
8787
Annotations
88+
CaliperIntegration
8889
UpcomingFeatures
8990

9091
.. toctree::

‎src/c/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.12)
22

33
project(PerfFlowAspect VERSION "0.1.0")
44

5+
# Higher-level build options.
6+
option(PERFFLOWASPECT_WITH_CALIPER "Build with Caliper support" OFF)
7+
58
# Fail if using Clang < 18.0
69
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
710
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18.0)

‎src/c/cmake/Setup3rdParty.cmake‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,19 @@ include(cmake/thirdparty/FindOpenSSL.cmake)
1919
if(PERFFLOWASPECT_WITH_MULTITHREADS)
2020
include(cmake/thirdparty/FindThreads.cmake)
2121
endif()
22+
23+
if(PERFFLOWASPECT_WITH_CALIPER)
24+
# first Check for CALIPER_DIR
25+
if(NOT caliper_DIR)
26+
MESSAGE(FATAL_ERROR "Caliper support needs explicit caliper_DIR")
27+
endif()
28+
29+
if(caliper_DIR)
30+
message(STATUS "${caliper_DIR}")
31+
include(cmake/thirdparty/FindCaliper.cmake)
32+
endif()
33+
34+
if(CALIPER_FOUND)
35+
add_definitions(-DPERFFLOWASPECT_WITH_CALIPER)
36+
endif()
37+
endif()

‎src/c/cmake/thirdparty/FindCaliper.cmake‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
message(STATUS "Looking for Caliper in: ${caliper_DIR}")
2+
3+
find_package(caliper REQUIRED
4+
PATHS ${caliper_DIR}/share/cmake/caliper
5+
NO_DEFAULT_PATH
6+
NO_CMAKE_ENVIRONMENT_PATH
7+
NO_CMAKE_PATH
8+
NO_SYSTEM_ENVIRONMENT_PATH
9+
NO_CMAKE_SYSTEM_PATH)
10+
11+
message(STATUS "FOUND Caliper: ${caliper_DIR}")
12+
13+
#set(ADIAK_FOUND TRUE)
14+
set(CALIPER_FOUND TRUE)
15+
set(PERFFLOWASPECT_CALIPER_ENABLED TRUE)

‎src/c/config/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ configure_package_config_file(
4242
install(FILES
4343
${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config.cmake
4444
${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config-version.cmake
45+
perfflowaspect_setup_deps.cmake
46+
perfflowaspect_setup_targets.cmake
4547
DESTINATION ${PERFFLOWASPECT_INSTALL_CMAKE_MODULE_DIR})
4648

4749
# Create pkg-config .pc file

‎src/c/config/perfflowaspect-config.cmake.in‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ if (NOT PERFFLOWASPECT_CONFIG_LOADED)
33
set(PERFFLOWASPECT_DIR "@CMAKE_INSTALL_PREFIX@")
44
set(PERFFLOWASPECT_LIB_DIR "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@")
55

6+
set(PERFFLOWASPECT_CALIPER_ENABLED "@PERFFLOWASPECT_CALIPER_ENABLED@")
7+
set(PERFFLOWASPECT_CALIPER_DIR "@caliper_DIR@")
8+
69
include(CMakeFindDependencyMacro)
710

811
find_dependency(OpenSSL REQUIRED)

‎src/c/config/perfflowaspect_setup_deps.cmake‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
include(CMakeFindDependencyMacro)
2+
3+
if (NOT caliper_DIR)
4+
set(caliper_DIR ${PERFFLOWASPECT_CALIPER_DIR})
5+
endif()
6+
7+
if(caliper_DIR)
8+
if(NOT PerfFlowAspect_FIND_QUIETLY)
9+
message(STATUS "PerfFlowAspect was built with Caliper Support")
10+
message(STATUS "Looking for Caliper at: ${caliper_DIR}/share/cmake/caliper")
11+
endif()
12+
13+
# find caliper
14+
find_package(caliper REQUIRED
15+
NO_DEFAULT_PATH
16+
PATHS ${caliper_DIR}/share/cmake/caliper)
17+
endif()

0 commit comments

Comments
 (0)