The data backup and recovery functions have been added

Author: wuxianrong

.github/workflows/pika.yml

@@ -12,83 +12,72 @@ env:
   ARTIFACT_PIKA_NAME: artifact-pika
 
 jobs:
+
   build_on_ubuntu:
-    # The CMake configure and build commands are platform-agnostic and should work equally well on Windows or Mac.
-    # You can convert this to a matrix build if you need cross-platform coverage.
-    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
-      - name: Free Disk Space (Ubuntu Host)
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /usr/local/lib/android
-          sudo rm -rf /opt/ghc
-          sudo rm -rf /opt/hostedtoolcache/CodeQL
-          sudo docker system prune -af
       - uses: actions/checkout@v4
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 1.19
-
-      - name: ccache
-        uses: hendrikmuhs/[email protected]
-        with:
-          key: ubuntu-latest
-
-      - name: Install Deps
+      - name: Build Docker image
         run: |
-          sudo apt-get update
-          sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler clang-tidy
-
-      - name: Configure CMake
-        # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
-        # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-        run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address -D CMAKE_C_COMPILER_LAUNCHER=ccache -D CMAKE_CXX_COMPILER_LAUNCHER=ccache
-
-      - name: Build
-        # Build your program with the given configuration
-        run: cmake --build build --config ${{ env.BUILD_TYPE }}
+          docker build -t pika-ci -f ci/Dockerfile .
 
-      - name: Cleanup
+      - name: Build Pika inside Docker
         run: |
-          rm -rf ./buildtrees
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: ${{ env.ARTIFACT_PIKA_NAME }}
-          path: ${{ github.workspace }}/build/pika
-
-      - name: Test
-        working-directory: ${{ github.workspace }}/build
-        # Execute tests defined by the CMake configuration.
-        # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
-        run: ctest -C ${{ env.BUILD_TYPE }}
+          docker run --rm \
+            -v ${{ github.workspace }}:/work \
+            -w /work \
+            pika-ci \
+            bash -c "rm -rf build buildtrees && \
+                     cmake -B build -DCMAKE_BUILD_TYPE=Release -DUSE_PIKA_TOOLS=ON && \
+                     cmake --build build -j2"
+
+      - name: Run CTest
+        run: |
+          docker run --rm \
+            -v ${{ github.workspace }}:/work \
+            -w /work/build \
+            pika-ci \
+            ctest --output-on-failure
 
-      - name: Unit Test
-        working-directory: ${{ github.workspace }}
-        run: ./pikatests.sh all clean
+      - name: Run Unit Tests
+        run: |
+          docker run --rm \
+            -v ${{ github.workspace }}:/work \
+            -w /work \
+            pika-ci \
+            ./pikatests.sh all clean
 
-        # master on port 9221, slave on port 9231, all with 2 db
-      - name: Start codis, pika master and pika slave
-        working-directory: ${{ github.workspace }}/build
+      - name: Start Codis and Pika Integration
         run: |
-          chmod +x ../tests/integration/start_master_and_slave.sh
-          ../tests/integration/start_master_and_slave.sh
-          chmod +x ../tests/integration/start_codis.sh
-          ../tests/integration/start_codis.sh
+          docker run --rm \
+            -v ${{ github.workspace }}:/work \
+            -w /work/build \
+            pika-ci \
+            bash -c "
+              chmod +x ../tests/integration/start_master_and_slave.sh &&
+              ../tests/integration/start_master_and_slave.sh &&
+              chmod +x ../tests/integration/start_codis.sh &&
+              ../tests/integration/start_codis.sh
+            "
 
       - name: Run Go E2E Tests
-        working-directory: ${{ github.workspace }}/build
         run: |
-          cd ../tools/pika_keys_analysis/
-          go test -v ./...
-          cd ../../tests/integration/
-          chmod +x integrate_test.sh
-          sh integrate_test.sh
-
+          docker run --rm \
+            -v ${{ github.workspace }}:/work \
+            -w /work/build \
+            pika-ci \
+            bash -c "
+              find . -name 'pika.conf' -exec sed -i 's/loglevel : info/loglevel : error/g' {} + &&
+              find . -name 'pika.conf' -exec sed -i 's/loglevel : debug/loglevel : error/g' {} + &&
+              cd ../tools/pika_keys_analysis/ &&
+              go test -v ./... &&
+              cd ../../tests/integration/ &&
+              chmod +x integrate_test.sh &&
+              sh integrate_test.sh
+            "
+             
   build_on_rocky:
     runs-on: ubuntu-latest
     container:
@@ -272,6 +261,7 @@ jobs:
           chmod +x integrate_test.sh
           sh integrate_test.sh
 
+
   build_on_macos:
 
     runs-on: macos-14

CMakeLists.txt

@@ -89,6 +89,18 @@ set(INSTALL_LIBDIR ${STAGED_INSTALL_PREFIX}/lib)
 set(INSTALL_LIBDIR_64 ${STAGED_INSTALL_PREFIX}/lib64)
 set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${STAGED_INSTALL_PREFIX})
 
+# Fix __const__ macro conflict with GCC 13+/glibc 2.38+ in existing brpc headers
+# This must run during CMake configuration to fix cached/existing headers
+if(EXISTS "${INSTALL_INCLUDEDIR}/butil/errno.h")
+  file(READ "${INSTALL_INCLUDEDIR}/butil/errno.h" ERRNO_H_CONTENT)
+  string(FIND "${ERRNO_H_CONTENT}" "#ifndef __const__" HAS_CONST_MACRO)
+  if(NOT HAS_CONST_MACRO EQUAL -1)
+    message(STATUS "Patching existing brpc errno.h for GCC 13+ compatibility")
+    string(REPLACE "#ifndef __const__" "#if 0 /* disabled for GCC 13+ compatibility */" ERRNO_H_CONTENT "${ERRNO_H_CONTENT}")
+    file(WRITE "${INSTALL_INCLUDEDIR}/butil/errno.h" "${ERRNO_H_CONTENT}")
+  endif()
+endif()
+
 execute_process(COMMAND sh ${CMAKE_UTILS_DIR}/Get_OS_Version.sh
   OUTPUT_VARIABLE OS_VERSION)
 
@@ -664,6 +676,12 @@ ExternalProject_Add(brpc
   1
   UPDATE_COMMAND
   ""
+  # Patch to fix __const__ macro conflict with GCC 13+/glibc 2.38+
+  # The issue is that brpc defines __const__ as __unused__ which conflicts with
+  # glibc's __glibc_has_attribute(__const__) macro in sys/cdefs.h
+  # We replace '#ifndef __const__' with '#if 0' to disable the problematic macro
+  PATCH_COMMAND
+  sh -c "sed -i.bak 's/#ifndef __const__/#if 0 /' src/butil/errno.h && cat src/butil/errno.h | head -35"
   LOG_CONFIGURE
   1
   LOG_BUILD
@@ -677,16 +695,29 @@ ExternalProject_Add(brpc
   -DCMAKE_INSTALL_PREFIX=${STAGED_INSTALL_PREFIX}
   -DCMAKE_BUILD_TYPE=${LIB_BUILD_TYPE}
   -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}
+  -DCMAKE_FIND_ROOT_PATH=${STAGED_INSTALL_PREFIX}
+  -DCMAKE_INCLUDE_PATH=${INSTALL_INCLUDEDIR}
+  -DCMAKE_LIBRARY_PATH=${INSTALL_LIBDIR}
   -DWITH_GLOG=ON
   -DWITH_SNAPPY=ON
   -DBUILD_SHARED_LIBS=OFF
   -DDOWNLOAD_GTEST=OFF
+  -DCMAKE_EXE_LINKER_FLAGS=-L${INSTALL_LIBDIR}\ -L${INSTALL_LIBDIR_64}
   BUILD_ALWAYS
   1
   BUILD_COMMAND
   make -j${CPU_CORE}
 )
 
+# Fix the installed header file after brpc installation
+# This is needed because braft uses the installed headers, not the patched source
+# The __const__ macro in butil/errno.h conflicts with glibc 2.38+ / GCC 13+
+ExternalProject_Add_Step(brpc fix_installed_headers
+  COMMAND sh -c "sed -i.bak 's/#ifndef __const__/#if 0 /' ${INSTALL_INCLUDEDIR}/butil/errno.h && cat ${INSTALL_INCLUDEDIR}/butil/errno.h | head -35"
+  DEPENDEES install
+  COMMENT "Patching installed brpc headers for GCC 13+ compatibility"
+)
+
 if(${OS_VERSION} MATCHES "Rocky" OR ${OS_VERSION} MATCHES "CentOS")
   set(BRPC_LIBRARY ${INSTALL_LIBDIR_64}/libbrpc.a)
 else()

ci/Dockerfile

@@ -1,22 +1,43 @@
-FROM ubuntu:22.04
+FROM ubuntu:20.04
 
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Asia/Shanghai
+
+# 设置 TMPDIR 到工作目录，避免 /tmp 空间不足
+ENV TMPDIR=/work/tmp
+RUN mkdir -p $TMPDIR
+
+# 安装基础工具 + gcc-9 + protobuf + cmake
 RUN apt-get update && apt-get install -y \
+    build-essential \
+    gcc-9 g++-9 \
+    git \
+    autoconf \
+    libssl-dev \
+    libprotobuf-dev \
+    libgflags-dev \
+    protobuf-compiler \
+    curl \
     ca-certificates \
-    rsync && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists /var/cache/apt/archives
-
-ENV PIKA=/pika \
-    PATH=${PIKA}:${PIKA}/bin:${PATH}
+    python3 \
+    && rm -rf /var/lib/apt/lists/*
 
-WORKDIR ${PIKA}
+# 安装新版 CMake >= 3.18
+RUN curl -fsSL https://github.com/Kitware/CMake/releases/download/v3.27.9/cmake-3.27.9-linux-x86_64.tar.gz \
+    | tar -xz -C /opt && \
+    ln -s /opt/cmake-3.27.9-linux-x86_64/bin/cmake /usr/local/bin/cmake
 
-COPY artifact/pika ${PIKA}/bin/pika
-COPY entrypoint.sh /entrypoint.sh
-COPY conf/pika.conf ${PIKA}/conf/pika.conf
+# Go 1.19
+RUN curl -fsSL https://go.dev/dl/go1.19.linux-amd64.tar.gz | tar -C /usr/local -xz
+ENV PATH=/usr/local/go/bin:$PATH
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENV CC=gcc-9
+ENV CXX=g++-9
 
-EXPOSE 9221
+WORKDIR /work
+COPY . /work
 
-CMD ["/pika/bin/pika", "-c", "/pika/conf/pika.conf"]
+# 构建 Pika + braft
+RUN mkdir -p build && cd build && \
+    cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-g0" && \
+    make -j2

docs/raft_implementation_review.md

@@ -0,0 +1,93 @@
+# Pika Raft 模式实现 Review 文档
+
+## 1. 概述
+
+Pika 的 Raft 模式基于 [braft](https://github.com/baidu/braft) 实现分布式一致性，使用 brpc 进行节点间通信。
+
+**核心特性**：
+- 强一致性保证
+- 自动 Leader 选举
+- 基于 RocksDB Checkpoint 的快照机制
+
+## 2. 架构概览
+
+```
+┌─────────────────────────────────────────────────────┐
+│                    Pika Server                       │
+├─────────────────────────────────────────────────────┤
+│  Client Request → RaftManager → PikaRaftNode        │
+│                                    ↓                 │
+│                              braft::Node             │
+│                                    ↓                 │
+│                           PikaStateMachine          │
+│                                    ↓                 │
+│                         Storage::OnBinlogWrite()    │
+│                                    ↓                 │
+│                              RocksDB                 │
+└─────────────────────────────────────────────────────┘
+```
+
+## 3. 核心组件
+
+| 组件 | 文件 | 职责 |
+|------|------|------|
+| `RaftManager` | [praft.h](../src/praft/include/praft/praft.h) | 管理多个 Raft 节点（每个 DB 一个） |
+| `PikaRaftNode` | [praft.h](../src/praft/include/praft/praft.h) | 封装 braft::Node，提供日志追加接口 |
+| `PikaStateMachine` | [praft.cc](../src/praft/src/praft.cc) | 实现状态机，处理日志应用和快照 |
+| `PPosixFileSystemAdaptor` | [psnapshot.cc](../src/praft/src/psnapshot.cc) | 快照文件系统适配器 |
+
+## 4. 数据流
+
+### 写入流程
+1. 客户端发送写命令
+2. Storage 层构建 Binlog（Protobuf 格式）
+3. 通过 `PikaRaftNode::AppendLog()` 提交到 Raft
+4. braft 复制日志到多数节点
+5. 提交后 `PikaStateMachine::on_apply()` 被调用
+6. 调用 `Storage::OnBinlogWrite()` 写入 RocksDB
+
+### Binlog 格式
+
+定义在 [binlog.proto](../src/praft/src/binlog.proto)：
+- 支持数据类型：Strings, Hashes, Lists, Sets, ZSets, Streams
+- 操作类型：Put, Delete
+
+## 5. 快照机制
+
+- 使用 RocksDB Checkpoint 创建快照
+- 快照恢复点基于 `GetSmallestFlushedLogIndex()`
+- 通过 `LogIndexOfColumnFamilies` 追踪各 CF 的日志应用进度
+
+## 6. 配置选项
+
+| 配置项 | 默认值 | 说明 |
+|--------|--------|------|
+| `raft_enabled` | false | 是否启用 Raft |
+| `raft_group_id` | "" | Raft Group ID |
+| `raft_election_timeout_ms` | 1000 | 选举超时(ms) |
+| `raft_snapshot_interval_s` | 3600 | 快照间隔(s) |
+
+## 7. Review 重点
+
+### 正确性
+- **日志幂等性**: `IsApplied()` 检查是否已应用，防止重复应用
+- **快照一致性**: 使用最小 flushed_log_index 确定快照点
+- **WAL 禁用**: Raft 日志提供持久性保证，RocksDB WAL 被禁用
+
+### 性能
+- 同步写入等待 Raft 提交（可优化为批量提交）
+- Binlog 使用 Protobuf 序列化
+
+### 线程安全
+- `LogIndexOfColumnFamilies` 使用 mutex 保护
+- braft 保证 `on_apply()` 顺序调用
+
+## 8. 关键代码位置
+
+| 功能 | 文件 |
+|------|------|
+| 日志应用 | [praft.cc](../src/praft/src/praft.cc) - `on_apply()` |
+| Binlog 处理 | [storage.cc](../src/storage/src/storage.cc) - `OnBinlogWrite()` |
+| Log Index 追踪 | [log_index.cc](../src/storage/src/log_index.cc) |
+| 快照创建 | [psnapshot.cc](../src/praft/src/psnapshot.cc) |
+| Raft 命令 | [pika_raft.cc](../src/pika_raft.cc) |

include/pika_db.h

@@ -94,6 +94,8 @@ class DB : public std::enable_shared_from_this<DB>, public pstd::noncopyable {
   std::shared_ptr<storage::Storage> storage() const;
   void GetBgSaveMetaData(std::vector<std::string>* fileNames, std::string* snapshot_uuid);
   void BgSaveDB();
+  pstd::Status CreateCheckpoint(const std::string& checkpoint_dir);
+  pstd::Status LoadDBFromCheckpoint(const std::string& checkpoint_dir);
   void SetBinlogIoError();
   void SetBinlogIoErrorrelieve();
   bool IsBinlogIoError();

include/pika_server.h

@@ -71,6 +71,8 @@ enum TaskType {
   kCompactRangeSets,
   kCompactRangeZSets,
   kCompactRangeList,
+  kLoadDBFromCheckpoint,
+  kCreateCheckpoint,
 };
 
 struct TaskArg {