From acd4109d5952b22f10c236649b8d2f542ee31b9f Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Mon, 22 Jun 2026 14:31:51 +0000
Subject: [PATCH 01/19] feat(import): add snapshot import helper script and
 usage documentation

---
 .../SNAPSHOT_IMPORT_USAGE.md                  |  65 +++++++
 .../run-snapshot-import.sh                    | 171 ++++++++++++++++++
 2 files changed, 236 insertions(+)
 create mode 100644 bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
 create mode 100755 bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
new file mode 100644
index 0000000000..f90feef18a
--- /dev/null
+++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
@@ -0,0 +1,65 @@
+# HBase Snapshot Import Helper Script Usage
+
+This document describes the environment variables used by the `run-snapshot-import.sh` script to automate HBase snapshot imports into Cloud Bigtable using Dataflow.
+
+## Environment Variables
+
+The script relies on the following environment variables. You should set them before executing the script.
+
+| Variable | Description | Example / Suggested Value |
+| :--- | :--- | :--- |
+| `PROJECT_ID` | The Google Cloud Project ID where the Bigtable instance and Dataflow jobs reside. | `your-project-id` |
+| `INSTANCE_ID` | The Bigtable Instance ID to import data into. | `your-instance-id` |
+| `BUCKET` | The GCS bucket name used for Dataflow staging, temp files, and default snapshot source path. | `your-gcs-bucket` |
+| `REGION` | The GCP region to run the Dataflow jobs in. | `us-central1` |
+| `TABLE_NAME` | The target Bigtable table name. | `your-table-name` |
+| `SNAPSHOT_NAME` | The name of the HBase snapshot to import. | `your-snapshot-name` |
+| `SNAPSHOT_SOURCE_DIR` | The GCS path where the HBase snapshot export is located. | `gs://your-gcs-bucket/snapshots` |
+| `SERVICE_ACCOUNT` | The service account email to run the Dataflow jobs. | `your-service-account@developer.gserviceaccount.com` |
+| `NUM_SHARDS` | The number of shards to split the import into for parallel processing. | `20` |
+| `MAX_INFLIGHT_RPCS` | Maximum number of inflight RPCs for Bigtable client. | `100` |
+| `BULK_MUTATION_CLOSE_TIMEOUT_MINUTES` | Timeout in minutes for closing bulk mutations. | `30` |
+| `NETWORK` | VPC Network name for Dataflow workers. | `your-network` |
+| `SUBNETWORK` | VPC Subnetwork name for Dataflow workers. | `regions/us-central1/subnetworks/your-subnetwork` |
+
+## Usage
+
+### Run a specific shard range
+```bash
+./run-snapshot-import.sh <start_shard> <end_shard>
+```
+Example: `./run-snapshot-import.sh 0 5`
+
+### Run all shards (Auto-parallel mode)
+```bash
+./run-snapshot-import.sh --all
+```
+This mode will first run the restore step, and then launch background processes for all shards in parallel groups of 4 by default.
+
+## Advanced Usage
+
+### Manual Parallel Execution
+
+To run shards in parallel groups (e.g., assuming 20 shards total), you can run multiple instances of this script.
+
+> [!IMPORTANT]
+> Because concurrent shards cannot delete or overwrite the restored snapshot directory simultaneously, **no shard** performs the restore step during a sharded run. You MUST run the restore step explicitly first!
+
+Example for manual parallel execution:
+```bash
+# 1. Run the blocking restore step first!
+./run-snapshot-import.sh --restore-only
+
+# 2. Once the restore is complete, launch shards in parallel:
+./run-snapshot-import.sh 0 3 &
+./run-snapshot-import.sh 4 7 &
+./run-snapshot-import.sh 8 11 &
+./run-snapshot-import.sh 12 15 &
+./run-snapshot-import.sh 16 19 &
+```
+
+## Troubleshooting
+
+### JDK Compatibility
+
+If you are running on a newer JDK (like Java 21 or 26) and hit ByteBuddy errors, you can add `-Dnet.bytebuddy.experimental=true` to the `java` command lines in the script.
diff --git a/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh
new file mode 100755
index 0000000000..5f642a67b7
--- /dev/null
+++ b/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh
@@ -0,0 +1,171 @@
+#!/bin/bash
+
+# ==============================================================================
+# HBase Snapshot Import Helper Script
+# ==============================================================================
+# This script runs a range of Dataflow snapshot import jobs sequentially or in parallel.
+# Must be executed from the 'bigtable-dataflow-parent/bigtable-beam-import' directory.
+#
+# For detailed usage and advanced options, see: SNAPSHOT_IMPORT_USAGE.md
+# ==============================================================================
+
+# ------------------------------------------------------------------------------
+# Environment Variables
+# ------------------------------------------------------------------------------
+# Most users will need to set these variables before running the script.
+# See SNAPSHOT_IMPORT_USAGE.md for details and expected values.
+
+# --- Required / Common Configurations ---
+# export PROJECT_ID="your-project-id"
+# export INSTANCE_ID="your-instance-id"
+# export BUCKET="your-gcs-bucket"
+# export REGION="us-central1"
+# 
+# export TABLE_NAME="your-table-name"
+# export SNAPSHOT_NAME="your-snapshot-name"
+# export SNAPSHOT_SOURCE_DIR="gs://your-gcs-bucket/snapshots"
+# export SERVICE_ACCOUNT="your-service-account"
+
+# --- Sharding & Tuning ---
+# export NUM_SHARDS="20"
+# export MAX_INFLIGHT_RPCS="100"
+# export BULK_MUTATION_CLOSE_TIMEOUT_MINUTES="30"
+
+# --- Network Configurations ---
+# export NETWORK="your-network"
+# export SUBNETWORK="your-subnetwork"
+
+# ------------------------------------------------------------------------------
+# Usage
+# ------------------------------------------------------------------------------
+# Usage: ./run-snapshot-import.sh <start_shard> <end_shard>
+#   Or:  ./run-snapshot-import.sh --all
+#        (Runs all shards in parallel groups of 4 by default)
+#
+# Examples:
+#   ./run-snapshot-import.sh 0 3
+#   ./run-snapshot-import.sh --all
+
+if [ "$#" -ne 2 ] && [ "$1" != "--all" ] && [ "$1" != "--restore-only" ]; then
+    echo "Usage: $0 <start_shard> <end_shard>"
+    echo "   Or: $0 --all"
+    echo "   Or: $0 --restore-only"
+    exit 1
+fi
+
+START_SHARD=$1
+END_SHARD=$2
+
+# Configurations
+JAR_PATH="target/bigtable-beam-import-2.18.2-SNAPSHOT-shaded.jar"
+RESTORE_DIR="${SNAPSHOT_SOURCE_DIR}/restore-${SNAPSHOT_NAME}"
+
+# --- RESTORE ONLY MODE ---
+if [ "$1" == "--restore-only" ]; then
+    echo "🚀 Performing snapshot restore (blocking)..."
+    java -jar ${JAR_PATH} importsnapshot \
+      --runner=DataflowRunner \
+      --project=${PROJECT_ID} \
+      --bigtableInstanceId=${INSTANCE_ID} \
+      --bigtableTableId=${TABLE_NAME} \
+      --hbaseSnapshotSourceDir=${SNAPSHOT_SOURCE_DIR} \
+      --snapshots=${SNAPSHOT_NAME}:${TABLE_NAME} \
+      --stagingLocation=gs://${BUCKET}/dataflow/staging \
+      --tempLocation=gs://${BUCKET}/dataflow/temp \
+      --region=${REGION} \
+      --performOnlyRestoreStep=true \
+      --restorePath=${RESTORE_DIR} \
+      --jobName="restore-job" \
+      --network=${NETWORK} \
+      --subnetwork=${SUBNETWORK}
+    echo "✅ Restore completed."
+    echo "⚠️ IMPORTANT: Please manually cleanup the restore path once validation succeeds:"
+    echo "   gsutil rm -r ${RESTORE_DIR}"
+    exit 0
+fi
+
+# --- AUTO-PARALLEL MODE ---
+if [ "$1" == "--all" ]; then
+    echo "🚀 Starting fully automated snapshot import..."
+    
+    # Step 1: Perform ONLY the restore step
+    echo "Step 1/2: Performing snapshot restore (blocking)..."
+    java -jar ${JAR_PATH} importsnapshot \
+      --runner=DataflowRunner \
+      --project=${PROJECT_ID} \
+      --bigtableInstanceId=${INSTANCE_ID} \
+      --bigtableTableId=${TABLE_NAME} \
+      --hbaseSnapshotSourceDir=${SNAPSHOT_SOURCE_DIR} \
+      --snapshots=${SNAPSHOT_NAME}:${TABLE_NAME} \
+      --stagingLocation=gs://${BUCKET}/dataflow/staging \
+      --tempLocation=gs://${BUCKET}/dataflow/temp \
+      --region=${REGION} \
+      --performOnlyRestoreStep=true \
+      --restorePath=${RESTORE_DIR} \
+      --jobName="restore-job" \
+      --network=${NETWORK} \
+      --subnetwork=${SUBNETWORK}
+      
+    echo "Restore completed. Proceeding to data import."
+
+    # Step 2: Launch parallel groups of 4
+    echo "Step 2/2: Launching parallel groups of 4 shards..."
+    SHARDS_PER_GROUP=4
+    
+    for (( start=0; start<$NUM_SHARDS; start+=$SHARDS_PER_GROUP )); do
+        end=$((start + SHARDS_PER_GROUP - 1))
+        [ $end -ge $NUM_SHARDS ] && end=$((NUM_SHARDS - 1))
+        
+        echo "Launching group: shards $start to $end in background"
+        # Call ourselves with the range!
+        $0 $start $end &
+    done
+    
+    echo "All groups launched. Waiting for all background jobs to finish..."
+    wait
+    echo "🎉 All import jobs completed!"
+    echo "⚠️ IMPORTANT: Please manually cleanup the restore path once validation succeeds:"
+    echo "   gsutil rm -r ${RESTORE_DIR}"
+    exit 0
+fi
+# ----------------------------------------
+
+# Standard Range Mode
+for i in $(seq $START_SHARD $END_SHARD); do
+  echo "Submitting Dataflow job for shardIndex: $i"
+
+  # As per the sharding contract, ALL parallel sharded jobs MUST skip the restore step
+  # to prevent concurrent shards from deleting the restore path.
+  # The --all mode runs performOnlyRestoreStep=true automatically in Step 1.
+  SKIP_RESTORE="true"
+  
+  JOB="job-${i}"
+  java -jar ${JAR_PATH} importsnapshot \
+    --runner=DataflowRunner \
+    --project=${PROJECT_ID} \
+    --bigtableInstanceId=${INSTANCE_ID} \
+    --bigtableTableId=${TABLE_NAME} \
+    --hbaseSnapshotSourceDir=${SNAPSHOT_SOURCE_DIR} \
+    --snapshots=${SNAPSHOT_NAME}:${TABLE_NAME} \
+    --stagingLocation=gs://${BUCKET}/dataflow/staging \
+    --tempLocation=gs://${BUCKET}/dataflow/temp \
+    --workerMachineType=n1-highmem-4 \
+    --diskSizeGb=500 \
+    --maxNumWorkers=10 \
+    --region=${REGION} \
+    --serviceAccount=${SERVICE_ACCOUNT} \
+    --usePublicIps=false \
+    --enableSnappy=true \
+    --skipRestoreStep=${SKIP_RESTORE} \
+    --deleteRestoredSnapshots=false \
+    --restorePath=${RESTORE_DIR} \
+    --numShards=${NUM_SHARDS} \
+    --shardIndex=$i \
+    --jobName="${JOB}" \
+    --network=${NETWORK} \
+    --subnetwork=${SUBNETWORK} \
+    --maxInflightRpcs=${MAX_INFLIGHT_RPCS} \
+    --bulkMutationCloseTimeoutMinutes=${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES}
+
+  # Sequential within this script instance
+done

From 1e2551c56c17b0078b89d351f78309cf85901f57 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Tue, 23 Jun 2026 17:21:45 +0000
Subject: [PATCH 02/19] Fix JDK 21+ compatibility, restore path directory
 constraints, and network argument handling in snapshot import script

---
 .../bigtable-beam-import/pom.xml              |  1 -
 .../run-snapshot-import.sh                    | 39 +++++++++++++------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/pom.xml b/bigtable-dataflow-parent/bigtable-beam-import/pom.xml
index 69a6b52500..9edfd41a5b 100644
--- a/bigtable-dataflow-parent/bigtable-beam-import/pom.xml
+++ b/bigtable-dataflow-parent/bigtable-beam-import/pom.xml
@@ -334,7 +334,6 @@ limitations under the License.
 		      <exclude>META-INF/*.RSA</exclude>
           <exclude>META-INF/**/pom.properties</exclude>
           <exclude>META-INF/**/pom.xml</exclude>
-          <exclude>META-INF/MANIFEST.MF</exclude>
           <exclude>META-INF/LICENSE</exclude>
           <exclude>META-INF/NOTICE.txt</exclude>
           <exclude>META-INF/NOTICE</exclude>
diff --git a/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh
index 5f642a67b7..39c5631ef2 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh
@@ -58,12 +58,32 @@ END_SHARD=$2
 
 # Configurations
 JAR_PATH="target/bigtable-beam-import-2.18.2-SNAPSHOT-shaded.jar"
-RESTORE_DIR="${SNAPSHOT_SOURCE_DIR}/restore-${SNAPSHOT_NAME}"
+RESTORE_DIR="gs://${BUCKET}/restore-${SNAPSHOT_NAME}"
+# If running the launcher on a newer, unsupported JDK version (e.g. JDK 25+),
+# you can set JVM_OPTS to "-Dnet.bytebuddy.experimental=true" to bypass
+# ByteBuddy class generation crashes during pipeline construction.
+JVM_OPTS=""
+
+# Construct optional network arguments
+NETWORK_ARGS=()
+if [ -n "${NETWORK}" ]; then
+    NETWORK_ARGS+=("--network=${NETWORK}")
+fi
+if [ -n "${SUBNETWORK}" ]; then
+    NETWORK_ARGS+=("--subnetwork=${SUBNETWORK}")
+fi
+
+# Construct optional service account arguments
+SERVICE_ACCOUNT_ARGS=()
+if [ -n "${SERVICE_ACCOUNT}" ]; then
+    SERVICE_ACCOUNT_ARGS+=("--serviceAccount=${SERVICE_ACCOUNT}")
+fi
+
 
 # --- RESTORE ONLY MODE ---
 if [ "$1" == "--restore-only" ]; then
     echo "🚀 Performing snapshot restore (blocking)..."
-    java -jar ${JAR_PATH} importsnapshot \
+    java ${JVM_OPTS} -jar ${JAR_PATH} importsnapshot \
       --runner=DataflowRunner \
       --project=${PROJECT_ID} \
       --bigtableInstanceId=${INSTANCE_ID} \
@@ -76,8 +96,7 @@ if [ "$1" == "--restore-only" ]; then
       --performOnlyRestoreStep=true \
       --restorePath=${RESTORE_DIR} \
       --jobName="restore-job" \
-      --network=${NETWORK} \
-      --subnetwork=${SUBNETWORK}
+      "${NETWORK_ARGS[@]}"
     echo "✅ Restore completed."
     echo "⚠️ IMPORTANT: Please manually cleanup the restore path once validation succeeds:"
     echo "   gsutil rm -r ${RESTORE_DIR}"
@@ -90,7 +109,7 @@ if [ "$1" == "--all" ]; then
     
     # Step 1: Perform ONLY the restore step
     echo "Step 1/2: Performing snapshot restore (blocking)..."
-    java -jar ${JAR_PATH} importsnapshot \
+    java ${JVM_OPTS} -jar ${JAR_PATH} importsnapshot \
       --runner=DataflowRunner \
       --project=${PROJECT_ID} \
       --bigtableInstanceId=${INSTANCE_ID} \
@@ -103,8 +122,7 @@ if [ "$1" == "--all" ]; then
       --performOnlyRestoreStep=true \
       --restorePath=${RESTORE_DIR} \
       --jobName="restore-job" \
-      --network=${NETWORK} \
-      --subnetwork=${SUBNETWORK}
+      "${NETWORK_ARGS[@]}"
       
     echo "Restore completed. Proceeding to data import."
 
@@ -140,7 +158,7 @@ for i in $(seq $START_SHARD $END_SHARD); do
   SKIP_RESTORE="true"
   
   JOB="job-${i}"
-  java -jar ${JAR_PATH} importsnapshot \
+  java ${JVM_OPTS} -jar ${JAR_PATH} importsnapshot \
     --runner=DataflowRunner \
     --project=${PROJECT_ID} \
     --bigtableInstanceId=${INSTANCE_ID} \
@@ -153,7 +171,7 @@ for i in $(seq $START_SHARD $END_SHARD); do
     --diskSizeGb=500 \
     --maxNumWorkers=10 \
     --region=${REGION} \
-    --serviceAccount=${SERVICE_ACCOUNT} \
+    "${SERVICE_ACCOUNT_ARGS[@]}" \
     --usePublicIps=false \
     --enableSnappy=true \
     --skipRestoreStep=${SKIP_RESTORE} \
@@ -162,8 +180,7 @@ for i in $(seq $START_SHARD $END_SHARD); do
     --numShards=${NUM_SHARDS} \
     --shardIndex=$i \
     --jobName="${JOB}" \
-    --network=${NETWORK} \
-    --subnetwork=${SUBNETWORK} \
+    "${NETWORK_ARGS[@]}" \
     --maxInflightRpcs=${MAX_INFLIGHT_RPCS} \
     --bulkMutationCloseTimeoutMinutes=${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES}
 

From 38715e3c38d0cd9630526970108b5f3c7c3fbac5 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Tue, 23 Jun 2026 17:32:03 +0000
Subject: [PATCH 03/19] Update SNAPSHOT_IMPORT_USAGE.md troubleshooting section
 for JVM_OPTS

---
 .../bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
index f90feef18a..ab9d9f9fa5 100644
--- a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
+++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
@@ -62,4 +62,4 @@ Example for manual parallel execution:
 
 ### JDK Compatibility
 
-If you are running on a newer JDK (like Java 21 or 26) and hit ByteBuddy errors, you can add `-Dnet.bytebuddy.experimental=true` to the `java` command lines in the script.
+If you are running on a newer, unsupported JDK version (e.g. JDK 25+) and hit ByteBuddy errors, you can simply set the `JVM_OPTS` variable to `"-Dnet.bytebuddy.experimental=true"` in the Configurations section of the `run-snapshot-import.sh` script.

From 72066889dafcc43c533343ab02406c9015caa401 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 19:50:49 +0000
Subject: [PATCH 04/19] refactor(import): relocate helper script to bin/, make
 path dynamic, support extra args, and preserve JVM_OPTS

---
 .../SNAPSHOT_IMPORT_USAGE.md                  | 36 ++++++++++-----
 .../{ => bin}/run-snapshot-import.sh          | 45 ++++++++++++++++---
 2 files changed, 63 insertions(+), 18 deletions(-)
 rename bigtable-dataflow-parent/bigtable-beam-import/{ => bin}/run-snapshot-import.sh (82%)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
index ab9d9f9fa5..ae478ad12d 100644
--- a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
+++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
@@ -1,6 +1,6 @@
 # HBase Snapshot Import Helper Script Usage
 
-This document describes the environment variables used by the `run-snapshot-import.sh` script to automate HBase snapshot imports into Cloud Bigtable using Dataflow.
+This document describes the environment variables used by the `bin/run-snapshot-import.sh` script to automate HBase snapshot imports into Cloud Bigtable using Dataflow.
 
 ## Environment Variables
 
@@ -22,17 +22,31 @@ The script relies on the following environment variables. You should set them be
 | `NETWORK` | VPC Network name for Dataflow workers. | `your-network` |
 | `SUBNETWORK` | VPC Subnetwork name for Dataflow workers. | `regions/us-central1/subnetworks/your-subnetwork` |
 
+## Understanding Sharding
+
+An HBase snapshot consists of multiple physical **regions** (ordered chunks of table keys). To migrate extremely large tables efficiently, you can partition (shard) the regions across multiple independent Dataflow jobs.
+
+*   **How Sharding Works:** The import tool distributes regions uniformly across a total number of shards (`NUM_SHARDS`) by hashing each region's encoded name and taking it modulo `NUM_SHARDS`.
+*   **Targeted Shards:** A Dataflow job configured with a specific `shardIndex` only reads and imports the HBase regions assigned to that particular shard.
+*   **Why use Sharding:** Instead of running a single massive Dataflow job that might hit quota limits or VM memory constraints, you can split the import into multiple smaller, parallelizable jobs.
+
 ## Usage
 
 ### Run a specific shard range
+
+Runs a range of shard indices sequentially within the script execution.
+
+> [!NOTE]
+> Both `<start_shard>` and `<end_shard>` are **inclusive**. E.g., running `bin/run-snapshot-import.sh 0 5` will submit and execute Dataflow jobs for shards `0, 1, 2, 3, 4, and 5` sequentially.
+
 ```bash
-./run-snapshot-import.sh <start_shard> <end_shard>
+bin/run-snapshot-import.sh <start_shard> <end_shard>
 ```
-Example: `./run-snapshot-import.sh 0 5`
+Example: `bin/run-snapshot-import.sh 0 5`
 
 ### Run all shards (Auto-parallel mode)
 ```bash
-./run-snapshot-import.sh --all
+bin/run-snapshot-import.sh --all
 ```
 This mode will first run the restore step, and then launch background processes for all shards in parallel groups of 4 by default.
 
@@ -48,18 +62,18 @@ To run shards in parallel groups (e.g., assuming 20 shards total), you can run m
 Example for manual parallel execution:
 ```bash
 # 1. Run the blocking restore step first!
-./run-snapshot-import.sh --restore-only
+bin/run-snapshot-import.sh --restore-only
 
 # 2. Once the restore is complete, launch shards in parallel:
-./run-snapshot-import.sh 0 3 &
-./run-snapshot-import.sh 4 7 &
-./run-snapshot-import.sh 8 11 &
-./run-snapshot-import.sh 12 15 &
-./run-snapshot-import.sh 16 19 &
+bin/run-snapshot-import.sh 0 3 &
+bin/run-snapshot-import.sh 4 7 &
+bin/run-snapshot-import.sh 8 11 &
+bin/run-snapshot-import.sh 12 15 &
+bin/run-snapshot-import.sh 16 19 &
 ```
 
 ## Troubleshooting
 
 ### JDK Compatibility
 
-If you are running on a newer, unsupported JDK version (e.g. JDK 25+) and hit ByteBuddy errors, you can simply set the `JVM_OPTS` variable to `"-Dnet.bytebuddy.experimental=true"` in the Configurations section of the `run-snapshot-import.sh` script.
+If you are running on a newer, unsupported JDK version (e.g. JDK 25+) and hit ByteBuddy errors, you can simply set the `JVM_OPTS` variable to `"-Dnet.bytebuddy.experimental=true"` in the Configurations section of the `bin/run-snapshot-import.sh` script.
diff --git a/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
similarity index 82%
rename from bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh
rename to bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 39c5631ef2..a34de350f1 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -4,7 +4,7 @@
 # HBase Snapshot Import Helper Script
 # ==============================================================================
 # This script runs a range of Dataflow snapshot import jobs sequentially or in parallel.
-# Must be executed from the 'bigtable-dataflow-parent/bigtable-beam-import' directory.
+# Can be run from any directory, e.g., 'bin/run-snapshot-import.sh'.
 #
 # For detailed usage and advanced options, see: SNAPSHOT_IMPORT_USAGE.md
 # ==============================================================================
@@ -27,6 +27,8 @@
 # export SERVICE_ACCOUNT="your-service-account"
 
 # --- Sharding & Tuning ---
+# A 'shard' is a partition of HBase regions. Sharding allows splitting the import into
+# multiple independent Dataflow jobs running in parallel to speed up the migration.
 # export NUM_SHARDS="20"
 # export MAX_INFLIGHT_RPCS="100"
 # export BULK_MUTATION_CLOSE_TIMEOUT_MINUTES="30"
@@ -38,13 +40,14 @@
 # ------------------------------------------------------------------------------
 # Usage
 # ------------------------------------------------------------------------------
-# Usage: ./run-snapshot-import.sh <start_shard> <end_shard>
-#   Or:  ./run-snapshot-import.sh --all
+# Usage: bin/run-snapshot-import.sh <start_shard> <end_shard>
+#        (Both start and end shard indices are inclusive)
+#   Or:  bin/run-snapshot-import.sh --all
 #        (Runs all shards in parallel groups of 4 by default)
 #
 # Examples:
-#   ./run-snapshot-import.sh 0 3
-#   ./run-snapshot-import.sh --all
+#   bin/run-snapshot-import.sh 0 3
+#   bin/run-snapshot-import.sh --all
 
 if [ "$#" -ne 2 ] && [ "$1" != "--all" ] && [ "$1" != "--restore-only" ]; then
     echo "Usage: $0 <start_shard> <end_shard>"
@@ -57,12 +60,40 @@ START_SHARD=$1
 END_SHARD=$2
 
 # Configurations
-JAR_PATH="target/bigtable-beam-import-2.18.2-SNAPSHOT-shaded.jar"
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
+
+# Detect the shaded JAR dynamically in the target directory
+JAR_PATH=""
+for jar in "${SCRIPT_DIR}/../target"/bigtable-beam-import-*-shaded.jar; do
+    if [ -f "$jar" ]; then
+        JAR_PATH="$jar"
+        break
+    fi
+done
+
+# If the JAR doesn't exist, build it first
+if [ -z "${JAR_PATH}" ]; then
+    echo "📦 Shaded JAR not found. Building the project first using Maven..."
+    (cd "${SCRIPT_DIR}/.." && mvn clean package -DskipTests)
+    
+    # Re-detect the JAR after building
+    for jar in "${SCRIPT_DIR}/../target"/bigtable-beam-import-*-shaded.jar; do
+        if [ -f "$jar" ]; then
+            JAR_PATH="$jar"
+            break
+        fi
+    done
+fi
+
+if [ -z "${JAR_PATH}" ]; then
+    echo "❌ Error: Failed to find or build the shaded JAR in ${SCRIPT_DIR}/../target/"
+    exit 1
+fi
 RESTORE_DIR="gs://${BUCKET}/restore-${SNAPSHOT_NAME}"
 # If running the launcher on a newer, unsupported JDK version (e.g. JDK 25+),
 # you can set JVM_OPTS to "-Dnet.bytebuddy.experimental=true" to bypass
 # ByteBuddy class generation crashes during pipeline construction.
-JVM_OPTS=""
+JVM_OPTS="${JVM_OPTS:-}"
 
 # Construct optional network arguments
 NETWORK_ARGS=()

From 7d177218b313700e702a6fa2485e0ec9c533df81 Mon Sep 17 00:00:00 2001
From: tianlei2 <tianlei@google.com>
Date: Wed, 24 Jun 2026 15:55:19 -0400
Subject: [PATCH 05/19] Update
 bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 .../bin/run-snapshot-import.sh                 | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index a34de350f1..ca8ee877c0 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -59,6 +59,24 @@ fi
 START_SHARD=$1
 END_SHARD=$2
 
+# Validate required environment variables
+REQUIRED_VARS=(PROJECT_ID INSTANCE_ID BUCKET REGION TABLE_NAME SNAPSHOT_NAME SNAPSHOT_SOURCE_DIR)
+for var in "${REQUIRED_VARS[@]}"; do
+    if [ -z "${!var}" ]; then
+        echo "❌ Error: Environment variable $var is not set."
+        exit 1
+    fi
+done
+
+if [ "$1" != "--restore-only" ] && [ -z "${NUM_SHARDS}" ]; then
+    echo "❌ Error: Environment variable NUM_SHARDS is not set."
+    exit 1
+fi
+
+# Set default values for optional tuning parameters
+MAX_INFLIGHT_RPCS="${MAX_INFLIGHT_RPCS:-100}"
+BULK_MUTATION_CLOSE_TIMEOUT_MINUTES="${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES:-30}"
+
 # Configurations
 SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
 

From 883fcb5700148aca85c1e75cb3541687322f4caf Mon Sep 17 00:00:00 2001
From: tianlei2 <tianlei@google.com>
Date: Wed, 24 Jun 2026 15:56:55 -0400
Subject: [PATCH 06/19] Update
 bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 .../bigtable-beam-import/bin/run-snapshot-import.sh  | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index ca8ee877c0..c28bc6069b 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -177,14 +177,16 @@ if [ "$1" == "--all" ]; then
 
     # Step 2: Launch parallel groups of 4
     echo "Step 2/2: Launching parallel groups of 4 shards..."
-    SHARDS_PER_GROUP=4
+    MAX_CONCURRENCY=4
+    SHARDS_PER_RUNNER=$(( (NUM_SHARDS + MAX_CONCURRENCY - 1) / MAX_CONCURRENCY ))
     
-    for (( start=0; start<$NUM_SHARDS; start+=$SHARDS_PER_GROUP )); do
-        end=$((start + SHARDS_PER_GROUP - 1))
+    for (( runner=0; runner<MAX_CONCURRENCY; runner++ )); do
+        start=$(( runner * SHARDS_PER_RUNNER ))
+        end=$(( start + SHARDS_PER_RUNNER - 1 ))
+        [ $start -ge $NUM_SHARDS ] && break
         [ $end -ge $NUM_SHARDS ] && end=$((NUM_SHARDS - 1))
         
-        echo "Launching group: shards $start to $end in background"
-        # Call ourselves with the range!
+        echo "Launching runner $runner: shards $start to $end in background"
         $0 $start $end &
     done
     

From 8e2a3256e955a28f2a3d8f4be27899cafbfb31de Mon Sep 17 00:00:00 2001
From: tianlei2 <tianlei@google.com>
Date: Wed, 24 Jun 2026 15:58:23 -0400
Subject: [PATCH 07/19] Update
 bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 .../bigtable-beam-import/bin/run-snapshot-import.sh             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index c28bc6069b..50aac02654 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -200,7 +200,7 @@ fi
 # ----------------------------------------
 
 # Standard Range Mode
-for i in $(seq $START_SHARD $END_SHARD); do
+for (( i=START_SHARD; i<=END_SHARD; i++ )); do
   echo "Submitting Dataflow job for shardIndex: $i"
 
   # As per the sharding contract, ALL parallel sharded jobs MUST skip the restore step

From 5c606744427baa0e2f3ea709db125d3b1cdbef9e Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:01:12 +0000
Subject: [PATCH 08/19] feat(import): merge remote PR validations & concurrency
 optimizations with local EXTRA_ARGS forwarding

---
 .../bin/run-snapshot-import.sh                | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 50aac02654..8ae6fc7a9f 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -49,16 +49,22 @@
 #   bin/run-snapshot-import.sh 0 3
 #   bin/run-snapshot-import.sh --all
 
-if [ "$#" -ne 2 ] && [ "$1" != "--all" ] && [ "$1" != "--restore-only" ]; then
-    echo "Usage: $0 <start_shard> <end_shard>"
-    echo "   Or: $0 --all"
-    echo "   Or: $0 --restore-only"
-    exit 1
+# Validate and parse arguments
+if [ "$1" != "--all" ] && [ "$1" != "--restore-only" ]; then
+    if [ -z "$1" ] || [ -z "$2" ]; then
+        echo "Usage: $0 <start_shard> <end_shard> [additional_args...]"
+        echo "        (Both start and end shard indices are inclusive)"
+        echo "   Or: $0 --all [additional_args...]"
+        echo "   Or: $0 --restore-only [additional_args...]"
+        exit 1
+    fi
+    START_SHARD=$1
+    END_SHARD=$2
+    EXTRA_ARGS=("${@:3}")
+else
+    EXTRA_ARGS=("${@:2}")
 fi
 
-START_SHARD=$1
-END_SHARD=$2
-
 # Validate required environment variables
 REQUIRED_VARS=(PROJECT_ID INSTANCE_ID BUCKET REGION TABLE_NAME SNAPSHOT_NAME SNAPSHOT_SOURCE_DIR)
 for var in "${REQUIRED_VARS[@]}"; do
@@ -145,7 +151,8 @@ if [ "$1" == "--restore-only" ]; then
       --performOnlyRestoreStep=true \
       --restorePath=${RESTORE_DIR} \
       --jobName="restore-job" \
-      "${NETWORK_ARGS[@]}"
+      "${NETWORK_ARGS[@]}" \
+      "${EXTRA_ARGS[@]}"
     echo "✅ Restore completed."
     echo "⚠️ IMPORTANT: Please manually cleanup the restore path once validation succeeds:"
     echo "   gsutil rm -r ${RESTORE_DIR}"
@@ -171,7 +178,8 @@ if [ "$1" == "--all" ]; then
       --performOnlyRestoreStep=true \
       --restorePath=${RESTORE_DIR} \
       --jobName="restore-job" \
-      "${NETWORK_ARGS[@]}"
+      "${NETWORK_ARGS[@]}" \
+      "${EXTRA_ARGS[@]}"
       
     echo "Restore completed. Proceeding to data import."
 
@@ -187,7 +195,8 @@ if [ "$1" == "--all" ]; then
         [ $end -ge $NUM_SHARDS ] && end=$((NUM_SHARDS - 1))
         
         echo "Launching runner $runner: shards $start to $end in background"
-        $0 $start $end &
+        # Call ourselves with the range and propagate any extra arguments!
+        $0 $start $end "${EXTRA_ARGS[@]}" &
     done
     
     echo "All groups launched. Waiting for all background jobs to finish..."
@@ -233,7 +242,8 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do
     --jobName="${JOB}" \
     "${NETWORK_ARGS[@]}" \
     --maxInflightRpcs=${MAX_INFLIGHT_RPCS} \
-    --bulkMutationCloseTimeoutMinutes=${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES}
+    --bulkMutationCloseTimeoutMinutes=${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES} \
+    "${EXTRA_ARGS[@]}"
 
   # Sequential within this script instance
 done

From c137f46e8a25e5a5888e1213ca0f5dbdf3eebc62 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:02:44 +0000
Subject: [PATCH 09/19] docs(import): update SNAPSHOT_IMPORT_USAGE.md to
 document required vs optional variables and defaults

---
 .../SNAPSHOT_IMPORT_USAGE.md                  | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
index ae478ad12d..285aebcc1d 100644
--- a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
+++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
@@ -6,21 +6,21 @@ This document describes the environment variables used by the `bin/run-snapshot-
 
 The script relies on the following environment variables. You should set them before executing the script.
 
-| Variable | Description | Example / Suggested Value |
-| :--- | :--- | :--- |
-| `PROJECT_ID` | The Google Cloud Project ID where the Bigtable instance and Dataflow jobs reside. | `your-project-id` |
-| `INSTANCE_ID` | The Bigtable Instance ID to import data into. | `your-instance-id` |
-| `BUCKET` | The GCS bucket name used for Dataflow staging, temp files, and default snapshot source path. | `your-gcs-bucket` |
-| `REGION` | The GCP region to run the Dataflow jobs in. | `us-central1` |
-| `TABLE_NAME` | The target Bigtable table name. | `your-table-name` |
-| `SNAPSHOT_NAME` | The name of the HBase snapshot to import. | `your-snapshot-name` |
-| `SNAPSHOT_SOURCE_DIR` | The GCS path where the HBase snapshot export is located. | `gs://your-gcs-bucket/snapshots` |
-| `SERVICE_ACCOUNT` | The service account email to run the Dataflow jobs. | `your-service-account@developer.gserviceaccount.com` |
-| `NUM_SHARDS` | The number of shards to split the import into for parallel processing. | `20` |
-| `MAX_INFLIGHT_RPCS` | Maximum number of inflight RPCs for Bigtable client. | `100` |
-| `BULK_MUTATION_CLOSE_TIMEOUT_MINUTES` | Timeout in minutes for closing bulk mutations. | `30` |
-| `NETWORK` | VPC Network name for Dataflow workers. | `your-network` |
-| `SUBNETWORK` | VPC Subnetwork name for Dataflow workers. | `regions/us-central1/subnetworks/your-subnetwork` |
+| Variable | Status | Description | Default / Example |
+| :--- | :--- | :--- | :--- |
+| `PROJECT_ID` | **Required** | The Google Cloud Project ID where the Bigtable instance and Dataflow jobs reside. | `your-project-id` |
+| `INSTANCE_ID` | **Required** | The Bigtable Instance ID to import data into. | `your-instance-id` |
+| `BUCKET` | **Required** | The GCS bucket name used for Dataflow staging, temp files, and default snapshot source path. | `your-gcs-bucket` |
+| `REGION` | **Required** | The GCP region to run the Dataflow jobs in. | `us-central1` |
+| `TABLE_NAME` | **Required** | The target Bigtable table name. | `your-table-name` |
+| `SNAPSHOT_NAME` | **Required** | The name of the HBase snapshot to import. | `your-snapshot-name` |
+| `SNAPSHOT_SOURCE_DIR` | **Required** | The GCS path where the HBase snapshot export is located. | `gs://your-gcs-bucket/snapshots` |
+| `NUM_SHARDS` | **Required** (except `--restore-only`) | The number of shards to split the import into for parallel processing. | `20` |
+| `SERVICE_ACCOUNT` | *Optional* | The service account email to run the Dataflow jobs. | `your-service-account@developer.gserviceaccount.com` |
+| `MAX_INFLIGHT_RPCS` | *Optional* | Maximum number of inflight RPCs for Bigtable client. | `100` (Default) |
+| `BULK_MUTATION_CLOSE_TIMEOUT_MINUTES` | *Optional* | Timeout in minutes for closing bulk mutations. | `30` (Default) |
+| `NETWORK` | *Optional* | VPC Network name for Dataflow workers. | `your-network` |
+| `SUBNETWORK` | *Optional* | VPC Subnetwork name for Dataflow workers. | `regions/us-central1/subnetworks/your-subnetwork` |
 
 ## Understanding Sharding
 

From a4d3d13325fd23b351e476b3166f539a99296da8 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:07:49 +0000
Subject: [PATCH 10/19] style(import): update outdated sharding comment to
 reflect 4 parallel runner streams

---
 .../bigtable-beam-import/bin/run-snapshot-import.sh           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 8ae6fc7a9f..e9f7dc7778 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -183,8 +183,8 @@ if [ "$1" == "--all" ]; then
       
     echo "Restore completed. Proceeding to data import."
 
-    # Step 2: Launch parallel groups of 4
-    echo "Step 2/2: Launching parallel groups of 4 shards..."
+    # Step 2: Launch 4 parallel runner streams
+    echo "Step 2/2: Launching 4 parallel runner streams..."
     MAX_CONCURRENCY=4
     SHARDS_PER_RUNNER=$(( (NUM_SHARDS + MAX_CONCURRENCY - 1) / MAX_CONCURRENCY ))
     

From 5f032c759c58719f9cf9d4a18e2fea8662f372de Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:08:53 +0000
Subject: [PATCH 11/19] docs(import): update SNAPSHOT_IMPORT_USAGE.md to
 describe the 4 parallel runner streams model

---
 .../bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
index 285aebcc1d..11284c5667 100644
--- a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
+++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
@@ -48,7 +48,7 @@ Example: `bin/run-snapshot-import.sh 0 5`
 ```bash
 bin/run-snapshot-import.sh --all
 ```
-This mode will first run the restore step, and then launch background processes for all shards in parallel groups of 4 by default.
+This mode will first run the blocking restore step, and then partition the total shards evenly and launch exactly 4 parallel runner streams in the background to process them. This ensures that no more than 4 Dataflow jobs are ever running concurrently, protecting your Bigtable instance from write-throttling and staying safely within GCP resource quotas.
 
 ## Advanced Usage
 

From 45288bc2459504792327b044d03cac7abc375694 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:14:49 +0000
Subject: [PATCH 12/19] refactor(import): optimize JAR detection to always
 select the newest compiled file

---
 .../bin/run-snapshot-import.sh                | 27 ++++++-------------
 1 file changed, 8 insertions(+), 19 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index e9f7dc7778..4ccf4b9568 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -86,31 +86,20 @@ BULK_MUTATION_CLOSE_TIMEOUT_MINUTES="${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES:-30}"
 # Configurations
 SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
 
-# Detect the shaded JAR dynamically in the target directory
+# Detect the newest shaded JAR dynamically in the target directory
 JAR_PATH=""
-for jar in "${SCRIPT_DIR}/../target"/bigtable-beam-import-*-shaded.jar; do
+for jar in "${SCRIPT_DIR}"/../target/bigtable-beam-import-*-shaded.jar; do
     if [ -f "$jar" ]; then
-        JAR_PATH="$jar"
-        break
-    fi
-done
-
-# If the JAR doesn't exist, build it first
-if [ -z "${JAR_PATH}" ]; then
-    echo "📦 Shaded JAR not found. Building the project first using Maven..."
-    (cd "${SCRIPT_DIR}/.." && mvn clean package -DskipTests)
-    
-    # Re-detect the JAR after building
-    for jar in "${SCRIPT_DIR}/../target"/bigtable-beam-import-*-shaded.jar; do
-        if [ -f "$jar" ]; then
+        if [ -z "${JAR_PATH}" ] || [ "$jar" -nt "${JAR_PATH}" ]; then
             JAR_PATH="$jar"
-            break
         fi
-    done
-fi
+    fi
+done
 
 if [ -z "${JAR_PATH}" ]; then
-    echo "❌ Error: Failed to find or build the shaded JAR in ${SCRIPT_DIR}/../target/"
+    echo "❌ Error: Shaded JAR not found in ${SCRIPT_DIR}/../target/"
+    echo "   Please compile and package the project first by running:"
+    echo "   mvn clean package -DskipTests"
     exit 1
 fi
 RESTORE_DIR="gs://${BUCKET}/restore-${SNAPSHOT_NAME}"

From 555ba00ace208d47186436b17ae444c4fccdb4f9 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:31:42 +0000
Subject: [PATCH 13/19] fix(import): pass service account args to restore step
 and quote variables

---
 .../bin/run-snapshot-import.sh                | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 4ccf4b9568..1b2d63997e 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -127,19 +127,20 @@ fi
 # --- RESTORE ONLY MODE ---
 if [ "$1" == "--restore-only" ]; then
     echo "🚀 Performing snapshot restore (blocking)..."
-    java ${JVM_OPTS} -jar ${JAR_PATH} importsnapshot \
+    java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
       --runner=DataflowRunner \
-      --project=${PROJECT_ID} \
-      --bigtableInstanceId=${INSTANCE_ID} \
-      --bigtableTableId=${TABLE_NAME} \
-      --hbaseSnapshotSourceDir=${SNAPSHOT_SOURCE_DIR} \
-      --snapshots=${SNAPSHOT_NAME}:${TABLE_NAME} \
-      --stagingLocation=gs://${BUCKET}/dataflow/staging \
-      --tempLocation=gs://${BUCKET}/dataflow/temp \
-      --region=${REGION} \
+      --project="${PROJECT_ID}" \
+      --bigtableInstanceId="${INSTANCE_ID}" \
+      --bigtableTableId="${TABLE_NAME}" \
+      --hbaseSnapshotSourceDir="${SNAPSHOT_SOURCE_DIR}" \
+      --snapshots="${SNAPSHOT_NAME}:${TABLE_NAME}" \
+      --stagingLocation="gs://${BUCKET}/dataflow/staging" \
+      --tempLocation="gs://${BUCKET}/dataflow/temp" \
+      --region="${REGION}" \
       --performOnlyRestoreStep=true \
-      --restorePath=${RESTORE_DIR} \
+      --restorePath="${RESTORE_DIR}" \
       --jobName="restore-job" \
+      "${SERVICE_ACCOUNT_ARGS[@]}" \
       "${NETWORK_ARGS[@]}" \
       "${EXTRA_ARGS[@]}"
     echo "✅ Restore completed."
@@ -154,19 +155,20 @@ if [ "$1" == "--all" ]; then
     
     # Step 1: Perform ONLY the restore step
     echo "Step 1/2: Performing snapshot restore (blocking)..."
-    java ${JVM_OPTS} -jar ${JAR_PATH} importsnapshot \
+    java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
       --runner=DataflowRunner \
-      --project=${PROJECT_ID} \
-      --bigtableInstanceId=${INSTANCE_ID} \
-      --bigtableTableId=${TABLE_NAME} \
-      --hbaseSnapshotSourceDir=${SNAPSHOT_SOURCE_DIR} \
-      --snapshots=${SNAPSHOT_NAME}:${TABLE_NAME} \
-      --stagingLocation=gs://${BUCKET}/dataflow/staging \
-      --tempLocation=gs://${BUCKET}/dataflow/temp \
-      --region=${REGION} \
+      --project="${PROJECT_ID}" \
+      --bigtableInstanceId="${INSTANCE_ID}" \
+      --bigtableTableId="${TABLE_NAME}" \
+      --hbaseSnapshotSourceDir="${SNAPSHOT_SOURCE_DIR}" \
+      --snapshots="${SNAPSHOT_NAME}:${TABLE_NAME}" \
+      --stagingLocation="gs://${BUCKET}/dataflow/staging" \
+      --tempLocation="gs://${BUCKET}/dataflow/temp" \
+      --region="${REGION}" \
       --performOnlyRestoreStep=true \
-      --restorePath=${RESTORE_DIR} \
+      --restorePath="${RESTORE_DIR}" \
       --jobName="restore-job" \
+      "${SERVICE_ACCOUNT_ARGS[@]}" \
       "${NETWORK_ARGS[@]}" \
       "${EXTRA_ARGS[@]}"
       

From 403d35a9aa4477cbc85478416221390ae7439411 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:33:41 +0000
Subject: [PATCH 14/19] fix(import): track background job PIDs and propagate
 exit status failures

---
 .../bin/run-snapshot-import.sh                | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 1b2d63997e..1a208449f8 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -145,7 +145,7 @@ if [ "$1" == "--restore-only" ]; then
       "${EXTRA_ARGS[@]}"
     echo "✅ Restore completed."
     echo "⚠️ IMPORTANT: Please manually cleanup the restore path once validation succeeds:"
-    echo "   gsutil rm -r ${RESTORE_DIR}"
+    echo "   gsutil rm -r \"${RESTORE_DIR}\""
     exit 0
 fi
 
@@ -179,6 +179,7 @@ if [ "$1" == "--all" ]; then
     MAX_CONCURRENCY=4
     SHARDS_PER_RUNNER=$(( (NUM_SHARDS + MAX_CONCURRENCY - 1) / MAX_CONCURRENCY ))
     
+    pids=()
     for (( runner=0; runner<MAX_CONCURRENCY; runner++ )); do
         start=$(( runner * SHARDS_PER_RUNNER ))
         end=$(( start + SHARDS_PER_RUNNER - 1 ))
@@ -187,14 +188,26 @@ if [ "$1" == "--all" ]; then
         
         echo "Launching runner $runner: shards $start to $end in background"
         # Call ourselves with the range and propagate any extra arguments!
-        $0 $start $end "${EXTRA_ARGS[@]}" &
+        "${BASH_SOURCE[0]}" $start $end "${EXTRA_ARGS[@]}" &
+        pids+=($!)
     done
     
     echo "All groups launched. Waiting for all background jobs to finish..."
-    wait
+    failed=0
+    for pid in "${pids[@]}"; do
+        if ! wait "$pid"; then
+            failed=1
+        fi
+    done
+
+    if [ $failed -ne 0 ]; then
+        echo "❌ Error: One or more background import jobs failed."
+        exit 1
+    fi
+    
     echo "🎉 All import jobs completed!"
     echo "⚠️ IMPORTANT: Please manually cleanup the restore path once validation succeeds:"
-    echo "   gsutil rm -r ${RESTORE_DIR}"
+    echo "   gsutil rm -r \"${RESTORE_DIR}\""
     exit 0
 fi
 # ----------------------------------------

From 3c6befd24bef32da2ff9e14bc4c756ec1fa3406f Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:34:56 +0000
Subject: [PATCH 15/19] fix(import): double-quote variables in the standard
 range mode java command

---
 .../bin/run-snapshot-import.sh                | 30 +++++++++----------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 1a208449f8..93e8d49b6f 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -222,31 +222,31 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do
   SKIP_RESTORE="true"
   
   JOB="job-${i}"
-  java ${JVM_OPTS} -jar ${JAR_PATH} importsnapshot \
+  java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
     --runner=DataflowRunner \
-    --project=${PROJECT_ID} \
-    --bigtableInstanceId=${INSTANCE_ID} \
-    --bigtableTableId=${TABLE_NAME} \
-    --hbaseSnapshotSourceDir=${SNAPSHOT_SOURCE_DIR} \
-    --snapshots=${SNAPSHOT_NAME}:${TABLE_NAME} \
-    --stagingLocation=gs://${BUCKET}/dataflow/staging \
-    --tempLocation=gs://${BUCKET}/dataflow/temp \
+    --project="${PROJECT_ID}" \
+    --bigtableInstanceId="${INSTANCE_ID}" \
+    --bigtableTableId="${TABLE_NAME}" \
+    --hbaseSnapshotSourceDir="${SNAPSHOT_SOURCE_DIR}" \
+    --snapshots="${SNAPSHOT_NAME}:${TABLE_NAME}" \
+    --stagingLocation="gs://${BUCKET}/dataflow/staging" \
+    --tempLocation="gs://${BUCKET}/dataflow/temp" \
     --workerMachineType=n1-highmem-4 \
     --diskSizeGb=500 \
     --maxNumWorkers=10 \
-    --region=${REGION} \
+    --region="${REGION}" \
     "${SERVICE_ACCOUNT_ARGS[@]}" \
     --usePublicIps=false \
     --enableSnappy=true \
-    --skipRestoreStep=${SKIP_RESTORE} \
+    --skipRestoreStep="${SKIP_RESTORE}" \
     --deleteRestoredSnapshots=false \
-    --restorePath=${RESTORE_DIR} \
-    --numShards=${NUM_SHARDS} \
-    --shardIndex=$i \
+    --restorePath="${RESTORE_DIR}" \
+    --numShards="${NUM_SHARDS}" \
+    --shardIndex="${i}" \
     --jobName="${JOB}" \
     "${NETWORK_ARGS[@]}" \
-    --maxInflightRpcs=${MAX_INFLIGHT_RPCS} \
-    --bulkMutationCloseTimeoutMinutes=${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES} \
+    --maxInflightRpcs="${MAX_INFLIGHT_RPCS}" \
+    --bulkMutationCloseTimeoutMinutes="${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES}" \
     "${EXTRA_ARGS[@]}"
 
   # Sequential within this script instance

From 724dbcb908618601032c8f754b27e420578b0346 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 20:35:45 +0000
Subject: [PATCH 16/19] fix(import): sanitize BUCKET env variable by stripping
 gs:// prefix and trailing slashes

---
 .../bigtable-beam-import/bin/run-snapshot-import.sh           | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 93e8d49b6f..f05aba2fa4 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -79,6 +79,10 @@ if [ "$1" != "--restore-only" ] && [ -z "${NUM_SHARDS}" ]; then
     exit 1
 fi
 
+# Strip leading gs:// and trailing slashes from BUCKET for robust GCS path construction
+BUCKET="${BUCKET#gs://}"
+BUCKET="${BUCKET%/}"
+
 # Set default values for optional tuning parameters
 MAX_INFLIGHT_RPCS="${MAX_INFLIGHT_RPCS:-100}"
 BULK_MUTATION_CLOSE_TIMEOUT_MINUTES="${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES:-30}"

From 9677db086c47511713f888ba40f6cc791462efa8 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Wed, 24 Jun 2026 21:20:34 +0000
Subject: [PATCH 17/19] feat(import): make job prefix and execution parameters
 configurable, add validation, and update docs

---
 .../SNAPSHOT_IMPORT_USAGE.md                  |  6 +++
 .../bin/run-snapshot-import.sh                | 49 ++++++++++++++-----
 2 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
index 11284c5667..1112f26498 100644
--- a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
+++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
@@ -21,6 +21,12 @@ The script relies on the following environment variables. You should set them be
 | `BULK_MUTATION_CLOSE_TIMEOUT_MINUTES` | *Optional* | Timeout in minutes for closing bulk mutations. | `30` (Default) |
 | `NETWORK` | *Optional* | VPC Network name for Dataflow workers. | `your-network` |
 | `SUBNETWORK` | *Optional* | VPC Subnetwork name for Dataflow workers. | `regions/us-central1/subnetworks/your-subnetwork` |
+| `JOB_NAME_PREFIX` | *Optional* | Safe, unique prefix for all Dataflow jobs to prevent name collisions. | `import-<sanitized_table_name>` (Default) |
+| `WORKER_MACHINE_TYPE` | *Optional* | Compute Engine machine type for Dataflow workers. | `n1-highmem-4` (Default) |
+| `DISK_SIZE_GB` | *Optional* | Worker disk size in Gigabytes. | `500` (Default) |
+| `MAX_NUM_WORKERS` | *Optional* | Maximum number of active Dataflow workers per job. | `10` (Default) |
+| `USE_PUBLIC_IPS` | *Optional* | Whether Dataflow workers should have public IPs. | `false` (Default) |
+| `ENABLE_SNAPPY` | *Optional* | Whether to enable Snappy compression for transit files. | `true` (Default) |
 
 ## Understanding Sharding
 
diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index f05aba2fa4..26f90ffdd2 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -58,6 +58,14 @@ if [ "$1" != "--all" ] && [ "$1" != "--restore-only" ]; then
         echo "   Or: $0 --restore-only [additional_args...]"
         exit 1
     fi
+    if ! [[ "$1" =~ ^[0-9]+$ ]] || ! [[ "$2" =~ ^[0-9]+$ ]]; then
+        echo "❌ Error: Shard indices must be non-negative integers."
+        exit 1
+    fi
+    if [ "$1" -gt "$2" ]; then
+        echo "❌ Error: start_shard ($1) cannot be greater than end_shard ($2)."
+        exit 1
+    fi
     START_SHARD=$1
     END_SHARD=$2
     EXTRA_ARGS=("${@:3}")
@@ -74,9 +82,15 @@ for var in "${REQUIRED_VARS[@]}"; do
     fi
 done
 
-if [ "$1" != "--restore-only" ] && [ -z "${NUM_SHARDS}" ]; then
-    echo "❌ Error: Environment variable NUM_SHARDS is not set."
-    exit 1
+if [ "$1" != "--restore-only" ]; then
+    if [ -z "${NUM_SHARDS}" ]; then
+        echo "❌ Error: Environment variable NUM_SHARDS is not set."
+        exit 1
+    fi
+    if ! [[ "${NUM_SHARDS}" =~ ^[0-9]+$ ]] || [ "${NUM_SHARDS}" -le 0 ]; then
+        echo "❌ Error: NUM_SHARDS must be a positive integer."
+        exit 1
+    fi
 fi
 
 # Strip leading gs:// and trailing slashes from BUCKET for robust GCS path construction
@@ -87,6 +101,17 @@ BUCKET="${BUCKET%/}"
 MAX_INFLIGHT_RPCS="${MAX_INFLIGHT_RPCS:-100}"
 BULK_MUTATION_CLOSE_TIMEOUT_MINUTES="${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES:-30}"
 
+# Configurable Dataflow execution parameters
+WORKER_MACHINE_TYPE="${WORKER_MACHINE_TYPE:-n1-highmem-4}"
+DISK_SIZE_GB="${DISK_SIZE_GB:-500}"
+MAX_NUM_WORKERS="${MAX_NUM_WORKERS:-10}"
+USE_PUBLIC_IPS="${USE_PUBLIC_IPS:-false}"
+ENABLE_SNAPPY="${ENABLE_SNAPPY:-true}"
+
+# Generate a safe, unique job name prefix to prevent collisions
+SAFE_TABLE_NAME=$(echo "${TABLE_NAME}" | tr '[:upper:]' '[:lower:]' | tr '_' '-' | tr -cd '[:alnum:]-')
+JOB_NAME_PREFIX="${JOB_NAME_PREFIX:-import-${SAFE_TABLE_NAME}}"
+
 # Configurations
 SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)
 
@@ -143,7 +168,7 @@ if [ "$1" == "--restore-only" ]; then
       --region="${REGION}" \
       --performOnlyRestoreStep=true \
       --restorePath="${RESTORE_DIR}" \
-      --jobName="restore-job" \
+      --jobName="${JOB_NAME_PREFIX}-restore" \
       "${SERVICE_ACCOUNT_ARGS[@]}" \
       "${NETWORK_ARGS[@]}" \
       "${EXTRA_ARGS[@]}"
@@ -171,7 +196,7 @@ if [ "$1" == "--all" ]; then
       --region="${REGION}" \
       --performOnlyRestoreStep=true \
       --restorePath="${RESTORE_DIR}" \
-      --jobName="restore-job" \
+      --jobName="${JOB_NAME_PREFIX}-restore" \
       "${SERVICE_ACCOUNT_ARGS[@]}" \
       "${NETWORK_ARGS[@]}" \
       "${EXTRA_ARGS[@]}"
@@ -192,7 +217,7 @@ if [ "$1" == "--all" ]; then
         
         echo "Launching runner $runner: shards $start to $end in background"
         # Call ourselves with the range and propagate any extra arguments!
-        "${BASH_SOURCE[0]}" $start $end "${EXTRA_ARGS[@]}" &
+        bash "${BASH_SOURCE[0]}" "$start" "$end" "${EXTRA_ARGS[@]}" &
         pids+=($!)
     done
     
@@ -225,7 +250,7 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do
   # The --all mode runs performOnlyRestoreStep=true automatically in Step 1.
   SKIP_RESTORE="true"
   
-  JOB="job-${i}"
+  JOB="${JOB_NAME_PREFIX}-${i}"
   java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
     --runner=DataflowRunner \
     --project="${PROJECT_ID}" \
@@ -235,13 +260,13 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do
     --snapshots="${SNAPSHOT_NAME}:${TABLE_NAME}" \
     --stagingLocation="gs://${BUCKET}/dataflow/staging" \
     --tempLocation="gs://${BUCKET}/dataflow/temp" \
-    --workerMachineType=n1-highmem-4 \
-    --diskSizeGb=500 \
-    --maxNumWorkers=10 \
+    --workerMachineType="${WORKER_MACHINE_TYPE}" \
+    --diskSizeGb="${DISK_SIZE_GB}" \
+    --maxNumWorkers="${MAX_NUM_WORKERS}" \
     --region="${REGION}" \
     "${SERVICE_ACCOUNT_ARGS[@]}" \
-    --usePublicIps=false \
-    --enableSnappy=true \
+    --usePublicIps="${USE_PUBLIC_IPS}" \
+    --enableSnappy="${ENABLE_SNAPPY}" \
     --skipRestoreStep="${SKIP_RESTORE}" \
     --deleteRestoredSnapshots=false \
     --restorePath="${RESTORE_DIR}" \

From 8b699a33bf4517c93127b06d093e1e50b44c99f4 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Thu, 25 Jun 2026 16:21:48 +0000
Subject: [PATCH 18/19] fix(import): add exit status checks and shard index
 validation to run-snapshot-import.sh

---
 .../SNAPSHOT_IMPORT_USAGE.md                  |  3 ++-
 .../bin/run-snapshot-import.sh                | 27 ++++++++++++++-----
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
index 1112f26498..7e92aefe5c 100644
--- a/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
+++ b/bigtable-dataflow-parent/bigtable-beam-import/SNAPSHOT_IMPORT_USAGE.md
@@ -10,7 +10,7 @@ The script relies on the following environment variables. You should set them be
 | :--- | :--- | :--- | :--- |
 | `PROJECT_ID` | **Required** | The Google Cloud Project ID where the Bigtable instance and Dataflow jobs reside. | `your-project-id` |
 | `INSTANCE_ID` | **Required** | The Bigtable Instance ID to import data into. | `your-instance-id` |
-| `BUCKET` | **Required** | The GCS bucket name used for Dataflow staging, temp files, and default snapshot source path. | `your-gcs-bucket` |
+| `BUCKET` | **Required** | The GCS bucket name. The script automatically strips any leading gs:// or trailing slashes for robust path construction. | `your-gcs-bucket` |
 | `REGION` | **Required** | The GCP region to run the Dataflow jobs in. | `us-central1` |
 | `TABLE_NAME` | **Required** | The target Bigtable table name. | `your-table-name` |
 | `SNAPSHOT_NAME` | **Required** | The name of the HBase snapshot to import. | `your-snapshot-name` |
@@ -44,6 +44,7 @@ Runs a range of shard indices sequentially within the script execution.
 
 > [!NOTE]
 > Both `<start_shard>` and `<end_shard>` are **inclusive**. E.g., running `bin/run-snapshot-import.sh 0 5` will submit and execute Dataflow jobs for shards `0, 1, 2, 3, 4, and 5` sequentially.
+> Shard indices must be non-negative integers strictly less than `NUM_SHARDS` (i.e. in the range `[0, NUM_SHARDS)`). The script validates this range and will fail fast if violated.
 
 ```bash
 bin/run-snapshot-import.sh <start_shard> <end_shard>
diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index 26f90ffdd2..ddfaa21f7f 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -91,6 +91,12 @@ if [ "$1" != "--restore-only" ]; then
         echo "❌ Error: NUM_SHARDS must be a positive integer."
         exit 1
     fi
+    if [ "$1" != "--all" ]; then
+        if [ "${START_SHARD}" -ge "${NUM_SHARDS}" ] || [ "${END_SHARD}" -ge "${NUM_SHARDS}" ]; then
+            echo "❌ Error: Shard indices (${START_SHARD}, ${END_SHARD}) must be less than NUM_SHARDS (${NUM_SHARDS})."
+            exit 1
+        fi
+    fi
 fi
 
 # Strip leading gs:// and trailing slashes from BUCKET for robust GCS path construction
@@ -156,7 +162,7 @@ fi
 # --- RESTORE ONLY MODE ---
 if [ "$1" == "--restore-only" ]; then
     echo "🚀 Performing snapshot restore (blocking)..."
-    java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
+    if ! java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
       --runner=DataflowRunner \
       --project="${PROJECT_ID}" \
       --bigtableInstanceId="${INSTANCE_ID}" \
@@ -171,7 +177,10 @@ if [ "$1" == "--restore-only" ]; then
       --jobName="${JOB_NAME_PREFIX}-restore" \
       "${SERVICE_ACCOUNT_ARGS[@]}" \
       "${NETWORK_ARGS[@]}" \
-      "${EXTRA_ARGS[@]}"
+      "${EXTRA_ARGS[@]}"; then
+        echo "❌ Error: Snapshot restore failed."
+        exit 1
+    fi
     echo "✅ Restore completed."
     echo "⚠️ IMPORTANT: Please manually cleanup the restore path once validation succeeds:"
     echo "   gsutil rm -r \"${RESTORE_DIR}\""
@@ -184,7 +193,7 @@ if [ "$1" == "--all" ]; then
     
     # Step 1: Perform ONLY the restore step
     echo "Step 1/2: Performing snapshot restore (blocking)..."
-    java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
+    if ! java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
       --runner=DataflowRunner \
       --project="${PROJECT_ID}" \
       --bigtableInstanceId="${INSTANCE_ID}" \
@@ -199,7 +208,10 @@ if [ "$1" == "--all" ]; then
       --jobName="${JOB_NAME_PREFIX}-restore" \
       "${SERVICE_ACCOUNT_ARGS[@]}" \
       "${NETWORK_ARGS[@]}" \
-      "${EXTRA_ARGS[@]}"
+      "${EXTRA_ARGS[@]}"; then
+        echo "❌ Error: Snapshot restore failed. Aborting import."
+        exit 1
+    fi
       
     echo "Restore completed. Proceeding to data import."
 
@@ -251,7 +263,7 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do
   SKIP_RESTORE="true"
   
   JOB="${JOB_NAME_PREFIX}-${i}"
-  java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
+  if ! java ${JVM_OPTS} -jar "${JAR_PATH}" importsnapshot \
     --runner=DataflowRunner \
     --project="${PROJECT_ID}" \
     --bigtableInstanceId="${INSTANCE_ID}" \
@@ -276,7 +288,10 @@ for (( i=START_SHARD; i<=END_SHARD; i++ )); do
     "${NETWORK_ARGS[@]}" \
     --maxInflightRpcs="${MAX_INFLIGHT_RPCS}" \
     --bulkMutationCloseTimeoutMinutes="${BULK_MUTATION_CLOSE_TIMEOUT_MINUTES}" \
-    "${EXTRA_ARGS[@]}"
+    "${EXTRA_ARGS[@]}"; then
+      echo "❌ Error: Dataflow job submission failed for shardIndex: ${i}"
+      exit 1
+  fi
 
   # Sequential within this script instance
 done

From 29c9a85c94fa385c4fa66fffa72ce589c0436788 Mon Sep 17 00:00:00 2001
From: Tianlei Pan <tianlei@google.com>
Date: Thu, 25 Jun 2026 16:28:12 +0000
Subject: [PATCH 19/19] refactor(import): export env vars, add interruption
 cleanup trap, and use absolute path for child processes

---
 .../bigtable-beam-import/bin/run-snapshot-import.sh        | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
index ddfaa21f7f..0cf938e7ae 100755
--- a/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
+++ b/bigtable-dataflow-parent/bigtable-beam-import/bin/run-snapshot-import.sh
@@ -73,13 +73,14 @@ else
     EXTRA_ARGS=("${@:2}")
 fi
 
-# Validate required environment variables
+# Validate and export required environment variables
 REQUIRED_VARS=(PROJECT_ID INSTANCE_ID BUCKET REGION TABLE_NAME SNAPSHOT_NAME SNAPSHOT_SOURCE_DIR)
 for var in "${REQUIRED_VARS[@]}"; do
     if [ -z "${!var}" ]; then
         echo "❌ Error: Environment variable $var is not set."
         exit 1
     fi
+    export "${var}"
 done
 
 if [ "$1" != "--restore-only" ]; then
@@ -91,6 +92,7 @@ if [ "$1" != "--restore-only" ]; then
         echo "❌ Error: NUM_SHARDS must be a positive integer."
         exit 1
     fi
+    export NUM_SHARDS
     if [ "$1" != "--all" ]; then
         if [ "${START_SHARD}" -ge "${NUM_SHARDS}" ] || [ "${END_SHARD}" -ge "${NUM_SHARDS}" ]; then
             echo "❌ Error: Shard indices (${START_SHARD}, ${END_SHARD}) must be less than NUM_SHARDS (${NUM_SHARDS})."
@@ -221,6 +223,7 @@ if [ "$1" == "--all" ]; then
     SHARDS_PER_RUNNER=$(( (NUM_SHARDS + MAX_CONCURRENCY - 1) / MAX_CONCURRENCY ))
     
     pids=()
+    trap 'echo "⚠️ Interrupted. Terminating background runners..."; [ ${#pids[@]} -gt 0 ] && kill "${pids[@]}" 2>/dev/null; exit 1' INT TERM
     for (( runner=0; runner<MAX_CONCURRENCY; runner++ )); do
         start=$(( runner * SHARDS_PER_RUNNER ))
         end=$(( start + SHARDS_PER_RUNNER - 1 ))
@@ -229,7 +232,7 @@ if [ "$1" == "--all" ]; then
         
         echo "Launching runner $runner: shards $start to $end in background"
         # Call ourselves with the range and propagate any extra arguments!
-        bash "${BASH_SOURCE[0]}" "$start" "$end" "${EXTRA_ARGS[@]}" &
+        bash "${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")" "$start" "$end" "${EXTRA_ARGS[@]}" &
         pids+=($!)
     done