feat: Enable batch import of SST files with S3/MinIO in Pika

.clang-format

@@ -0,0 +1,200 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: None
+AlignConsecutiveAssignments: None
+AlignConsecutiveBitFields: None
+AlignConsecutiveDeclarations: None
+AlignEscapedNewlines: Left
+AlignOperands:   Align
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: WithoutElse
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+AttributeMacros:
+  - __capability
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile:     false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeConceptDeclarations: true
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: true
+DisableFormat:   false
+EmptyLineBeforeAccessModifier: LogicalBlock
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+StatementAttributeLikeMacros:
+  - Q_EMIT
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^<ext/.*\.h>'
+    Priority:        2
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '^<.*'
+    Priority:        2
+    SortPriority:    0
+    CaseSensitive:   false
+  - Regex:           '.*'
+    Priority:        3
+    SortPriority:    0
+    CaseSensitive:   false
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IncludeIsMainSourceRegex: ''
+IndentCaseLabels: true
+IndentCaseBlocks: false
+IndentGotoLabels: true
+IndentPPDirectives: AfterHash
+IndentExternBlock: AfterExternBlock
+IndentRequires:  false
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+InsertTrailingCommas: None
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Never
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PenaltyIndentedWhitespace: 0
+PointerAlignment: Left
+RawStringFormats:
+  - Language:        Cpp
+    Delimiters:
+      - cc
+      - CC
+      - cpp
+      - Cpp
+      - CPP
+      - 'c++'
+      - 'C++'
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+  - Language:        TextProto
+    Delimiters:
+      - pb
+      - PB
+      - proto
+      - PROTO
+    EnclosingFunctions:
+      - EqualsProto
+      - EquivToProto
+      - PARSE_PARTIAL_TEXT_PROTO
+      - PARSE_TEST_PROTO
+      - PARSE_TEXT_PROTO
+      - ParseTextOrDie
+      - ParseTextProtoOrDie
+      - ParseTestProto
+      - ParsePartialTestProto
+    CanonicalDelimiter: ''
+    BasedOnStyle:    google
+ReflowComments:  true
+SortIncludes:    true
+SortJavaStaticImport: Before
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCaseColon: false
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceAroundPointerQualifiers: Default
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+BitFieldColonSpacing: Both
+Standard:        Auto
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        8
+UseCRLF:         false
+UseTab:          Never
+WhitespaceSensitiveMacros:
+  - STRINGIZE
+  - PP_STRINGIZE
+  - BOOST_PP_STRINGIZE
+  - NS_SWIFT_NAME
+  - CF_SWIFT_NAME
+...
+

.clang-tidy

@@ -0,0 +1,140 @@
+---
+Checks:     '
+            bugprone-*,
+            clang-analyzer-*,
+            google-*,
+            modernize-*,
+            performance-*,
+            portability-*,
+            -readability-*,
+            -cppcoreguidelines-*,
+            -bugprone-easily-swappable-parameters,
+            -bugprone-implicit-widening-of-multiplication-result,
+            -bugprone-reserved-identifier,
+            -bugprone-signed-char-misuse,
+            -bugprone-branch-clone,
+            -clang-analyzer-cplusplus.NewDelete,
+            -clang-analyzer-cplusplus.NewDeleteLeaks,
+            -clang-analyzer-security.insecureAPI.rand,
+            -clang-analyzer-cplusplus.InnerPointer,
+            -clang-analyzer-core.CallAndMessage,
+            -clang-analyzer-deadcode.DeadStores,
+            -clang-diagnostic-implicit-int-float-conversion,
+            -clang-diagnostic-unused-command-line-argument,
+            -clang-diagnostic-ignored-optimization-argument,
+            -google-readability-avoid-underscore-in-googletest-name,
+            -google-default-arguments,
+            -google-explicit-constructor,
+            -google-runtime-int,
+            -modernize-avoid-c-arrays,
+            -modernize-use-nodiscard,
+            -modernize-use-trailing-return-type,
+            -modernize-pass-by-value,
+            -modernize-concat-nested-namespaces,
+            -modernize-return-braced-init-list,
+			      -modernize-use-equals-delete,
+			      -modernize-redundant-void-arg,
+            -readability-convert-member-functions-to-static,
+            -readability-identifier-length,
+            -readability-function-cognitive-complexity,
+            -readability-magic-numbers,
+            -readability-make-member-function-const,
+            -readability-qualified-auto,
+            -readability-redundant-access-specifiers,
+            -readability-function-size,
+            -readability-else-after-return,
+            -readability-use-anyofallof,
+            -readability-implicit-bool-conversion,
+            -use_of_tag_name_without_tag,
+            '
+# naming check
+CheckOptions:
+  - { key: readability-identifier-naming.ClassCase,                value: CamelCase  }
+  - { key: readability-identifier-naming.EnumCase,                 value: CamelCase  }
+  - { key: readability-identifier-naming.FunctionCase,             value: CamelCase  }
+  - { key: readability-identifier-naming.NamespaceCase,            value: lower_case }
+  - { key: readability-identifier-naming.StructCase,               value: CamelCase  }
+  - { key: readability-identifier-naming.UnionCase,                value: CamelCase  }
+  - { key: readability-identifier-naming.TemplateParameterCase,    value: CamelCase  }
+  - { key: readability-identifier-naming.MacroDefinitionCase,      value: UPPER_CASE }
+
+  - { key: readability-identifier-naming.GlobalConstantCase,       value: CamelCase  }
+  - { key: readability-identifier-naming.GlobalConstantPrefix,     value: k          }
+  - { key: readability-identifier-naming.MemberConstantCase,       value: CamelCase  }
+  - { key: readability-identifier-naming.MemberConstantPrefix,     value: k          }
+  - { key: readability-identifier-naming.EnumConstantCase,         value: CamelCase  }
+  - { key: readability-identifier-naming.EnumConstantPrefix,       value: k          }
+  # - { key: readability-identifier-naming.StaticConstantCase,       value: CamelCase  }
+  # - { key: readability-identifier-naming.StaticConstantPrefix,     value: k          }
+  # - { key: readability-identifier-naming.ConstantCase,             value: CamelCase  }
+  # - { key: readability-identifier-naming.ConstantPrefix,           value: k          }
+  # - { key: readability-identifier-naming.ConstantParameterCase,     value: lower_case}
+
+  - { key: readability-identifier-naming.GlobalVariableCase,       value: lower_case }
+  - { key: readability-identifier-naming.GlobalVariablePrefix,     value: g_pika_    }
+  - { key: readability-identifier-naming.VariableCase,             value: lower_case }
+
+  - { key: readability-identifier-naming.PublicMethodCase,         value: CamelCase  }
+  - { key: readability-identifier-naming.PrivateMethodCase,        value: camelBack  }
+  - { key: readability-identifier-naming.ProtectedMethodCase,      value: camelBack  }
+  - { key: readability-identifier-naming.MemberCase,               value: lower_case }
+  - { key: readability-identifier-naming.MemberSuffix,             value: _          }
+
+WarningsAsErrors: '*'
+HeaderFilterRegex: '((pstd|net|storage)/include/.+\.h|include/pika_.+\.h)'
+AnalyzeTemporaryDtors: true
+
+#### Disabled checks and why: #####
+#
+# -readability-convert-member-functions-to-static,
+#     This check started going off in the upgrade from clang-tidy-8 to clang-tidy-12.  It is not always correct because 
+#     we hide the reference implementation in another repository.
+# -clang-analyzer-security.insecureAPI.rand, -clang-analyzer-security.insecureAPI.rand, -bugprone-unhandled-self-assignment, 
+# -bugprone-implicit-widening-of-multiplication-result
+#     These have not been investigated yet.
+# -bugprone-reserved-identifier, 
+#     Fails due to use of some __SHORT_FILE__ symbol, originating from very old code.
+# -bugprone-suspicious-include, 
+#     False positive due to GTest code.
+# -bugprone-too-small-loop-variable,
+#     Complains about uint8_t or uint16_t when the limit on the loop is a container's .size() (size_t).
+#     We usually do this when we know the maximum size of the container though, so propose leaving disabled.
+# -clang-analyzer-cplusplus.NewDelete,
+#     Seems to generate false positives. Suggest relying on ASAN and valgrind for memory stuff.
+# -clang-analyzer-cplusplus.NewDeleteLeaks,
+#     Seems to generate false positives. Suggest relying on ASAN and valgrind for memory stuff.
+# -modernize-use-nodiscard,
+#     New C++17 feature, slightly polarizing. Would clutter codebase.
+# -modernize-avoid-c-arrays,
+#     Let's not deal with people doing ridiculous things to hack around this. If it bites them, it bites them.
+# -bugprone-signed-char-misuse, -clang-diagnostic-implicit-int-float-conversion, -readability-make-member-function-const, 
+# -readability-qualified-auto, -readability-redundant-access-specifiers
+#     These were previously disabled for not being available in clang-tidy-8. They are now available on our clang-tidy-12, 
+#     and potentially worth investigating/fixing.
+# -bugprone-exception-escape
+#     Weird. No idea how to resolve.
+# -bugprone-branch-clone,
+#    frequent misidentification
+# -google-default-arguments
+#    Too many virtual functions in pika have default parameters, specifically void Do(std::shared_ptr<Partition> partition = nullptr)
+# -clang-analyzer-cplusplus.InnerPointer
+#     I don't know why this bug in the RocksDB library fails to suppress detection.So I have to disable this check.
+#--google-explicit-constructor
+#     There are so many implicit convert in pika
+#--use_of_tag_name_without_tag
+#     like class member is class , If there is no class tag, an error will be reported
+#     class A{  B b_;  } -> error ;  class A{  class B b_;  } -> no error ; 
+#-clang-analyzer-core.CallAndMessage
+#     pika/src/storage/src/scope_record_lock.h 
+#     Always detect a null pointer and call the TryLock method
+#     Called C++ object pointer is null [clang-analyzer-core.CallAndMessage,-warnings-as-errors]
+#     lock_mgr_->UnLock(key);
+#-clang-analyzer-deadcode.DeadStores
+#    error detection in some places
+#-google-runtime-int
+#    todo discussed, because there are so many of `long` and `long long` in pika, so disable for now
+#-clang-diagnostic-unused-command-line-argument
+#    because there are to many compile command error in github ci, but no error in local environment
+#    See in build folder compile_commands.json
+#-clang-diagnostic-ignored-optimization-argument
+#    The reason disabled this is same as above.