fix: optional Postgres via compose-with-db overlay

.github/actions/rhdh-local-compose-test/action.yaml

@@ -44,6 +44,14 @@ inputs:
     description: Log level to use
     required: false
     default: "info"
+  registry_redhat_io_username:
+    description: Username for registry.redhat.io (required when compose_config_name is with-db)
+    required: false
+    default: ""
+  registry_redhat_io_password:
+    description: Password or token for registry.redhat.io (required when compose_config_name is with-db)
+    required: false
+    default: ""
 
 runs:
   using: "composite"
@@ -180,6 +188,24 @@ runs:
         TOOL: ${{ inputs.container_tool }}
       run: $TOOL info
 
+    # Requires REGISTRY_REDHAT_IO_USERNAME / REGISTRY_REDHAT_IO_PASSWORD for with-db.
+    - name: Log in to registry.redhat.io
+      if: ${{ env.SKIP_TEST != 'true' && inputs.compose_config_name == 'with-db' }}
+      shell: bash
+      env:
+        TOOL: ${{ inputs.container_tool }}
+        REGISTRY_USER: ${{ inputs.registry_redhat_io_username }}
+        REGISTRY_PASSWORD: ${{ inputs.registry_redhat_io_password }}
+        PODMAN_CONTAINER: ${{ steps.setup-podman.outputs.container-name }}
+      run: |
+        echo "$REGISTRY_PASSWORD" | docker login registry.redhat.io -u "$REGISTRY_USER" --password-stdin
+        if [ "$TOOL" = "podman" ]; then
+          # Compose uses docker-compose inside the container, which reads Docker
+          # config there (podman login / host-only docker login are not enough).
+          docker exec "$PODMAN_CONTAINER" mkdir -p /root/.docker
+          docker cp "$HOME/.docker/config.json" "$PODMAN_CONTAINER:/root/.docker/config.json"
+        fi
+
     - name: Create .env file to override defaults
       if: ${{ env.SKIP_TEST != 'true' && inputs.override_images == 'true' }}
       shell: bash

.github/workflows/nightly.yaml

@@ -63,6 +63,10 @@ jobs:
             cliArgs: "-f compose.yaml -f compose-with-corporate-proxy.yaml"
           - name: "dynamic-plugins-root"
             cliArgs: "-f compose.yaml -f compose-dynamic-plugins-root.yaml"
+          # Requires REGISTRY_REDHAT_IO_USERNAME and REGISTRY_REDHAT_IO_PASSWORD
+          # (Red Hat registry service-account username + token).
+          - name: "with-db"
+            cliArgs: "-f compose.yaml -f compose-with-db.yaml"
           - name: "orchestrator-workflow"
             cliArgs: "-f compose.yaml -f orchestrator/compose.yaml"
           # TODO: Remove this and the exclude rules below once all supported
@@ -85,6 +89,10 @@ jobs:
             composeConfig:
               name: "dynamic-plugins-root"
               cliArgs: "-f compose.yaml -f compose-dynamic-plugins-root.yaml"
+          - os: ubuntu-24.04-arm
+            composeConfig:
+              name: "with-db"
+              cliArgs: "-f compose.yaml -f compose-with-db.yaml"
           - os: ubuntu-24.04-arm
             composeConfig:
               name: "developer-lightspeed"
@@ -126,3 +134,6 @@ jobs:
           compose_config_name: ${{ matrix.composeConfig.name }}
           user_config_enabled: ${{ matrix.userConfig }}
           override_images: "true"
+          registry_redhat_io_username: ${{ secrets.REGISTRY_REDHAT_IO_USERNAME }}
+          registry_redhat_io_password: ${{ secrets.REGISTRY_REDHAT_IO_PASSWORD }}
+

.github/workflows/test.yml

@@ -46,6 +46,10 @@ jobs:
             cliArgs: "-f compose.yaml -f compose-with-corporate-proxy.yaml"
           - name: "dynamic-plugins-root"
             cliArgs: "-f compose.yaml -f compose-dynamic-plugins-root.yaml"
+          # Requires REGISTRY_REDHAT_IO_USERNAME and REGISTRY_REDHAT_IO_PASSWORD
+          # (Red Hat registry service-account username + token).
+          - name: "with-db"
+            cliArgs: "-f compose.yaml -f compose-with-db.yaml"
 # TODO: re-enable this test once we can use the right references.
 # Tracked in https://redhat.atlassian.net/browse/RHDHBUGS-3559
 #          - name: "orchestrator-workflow"
@@ -69,3 +73,5 @@ jobs:
           compose_cli_args: ${{ matrix.composeConfig.cliArgs }}
           compose_config_name: ${{ matrix.composeConfig.name }}
           user_config_enabled: ${{ matrix.userConfig }}
+          registry_redhat_io_username: ${{ secrets.REGISTRY_REDHAT_IO_USERNAME }}
+          registry_redhat_io_password: ${{ secrets.REGISTRY_REDHAT_IO_PASSWORD }}

compose-with-db.yaml

 #            cliArgs: "-f compose.yaml -f orchestrator/compose.yaml" 
         composeConfig: 
           - name: "default" 
             cliArgs: "" 
           - name: "corporate-proxy" 
             cliArgs: "-f compose.yaml -f compose-with-corporate-proxy.yaml" 
           - name: "dynamic-plugins-root" 
             cliArgs: "-f compose.yaml -f compose-dynamic-plugins-root.yaml" 
 # TODO: re-enable this test once we can use the right references. 
 # Tracked in https://redhat.atlassian.net/browse/RHDHBUGS-3559 
 #          - name: "orchestrator-workflow" 
 #            cliArgs: "-f compose.yaml -f orchestrator/compose.yaml" 
@@ -0,0 +1,32 @@
+# This Compose file is not usable on its own.
+# It needs to be used alongside the default compose.yaml file,
+# since it adds a PostgreSQL db service and makes rhdh wait for it.
+#
+# You can run `[podman|docker] compose -f compose.yaml -f compose-with-db.yaml config`
+# to view the effective merged config.
+#
+# Also configure app-config.local.yaml for Postgres (see docs/rhdh-local-guide/postgresql-guide.md).
+
+services:
+  db:
+    container_name: db
+    image: "${POSTGRES_IMAGE:-registry.redhat.io/rhel10/postgresql-18:latest}" # dclint disable-line service-image-require-explicit-tag
+    volumes:
+      - "/var/lib/pgsql/data"
+    env_file:
+      - path: "./default.env"
+        required: true
+      - path: "./.env"
+        required: false
+    environment:
+      - POSTGRESQL_ADMIN_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+  rhdh:
+    depends_on:
+      db:
+        condition: service_healthy

compose.postgres-upgrade.override.example.yaml

@@ -0,0 +1,18 @@
+# Temporary override for a PostgreSQL major upgrade.
+# 1. Set POSTGRES_IMAGE in .env to the target major image (must support your
+#    current major via POSTGRESQL_PREV_VERSION).
+# 2. Copy this file to compose.override.yaml for a single upgrade boot.
+# 3. After upgrade succeeds, remove POSTGRESQL_UPGRADE (delete compose.override.yaml
+#    or strip that env) and recreate db.
+#
+# When using -f compose-with-db.yaml, Compose does not auto-load
+# compose.override.yaml — include it explicitly:
+#   podman compose -f compose.yaml -f compose-with-db.yaml -f compose.override.yaml up -d db
+#
+# See docs/rhdh-local-guide/postgresql-guide.md.
+
+services:
+  db:
+    environment:
+      - POSTGRESQL_ADMIN_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+      - POSTGRESQL_UPGRADE=copy

compose.yaml

@@ -1,25 +1,6 @@
 services:
-  # Uncomment the following block to use a PostgreSQL database
-  # don't forget to also uncomment 'depends_on' 'db' section in the rhdh service
-  # and comment out or delete 'database' section in app-config.yaml or app-config.local.yaml
-
-  # db:
-  #   container_name: db
-  #   image: "registry.redhat.io/rhel10/postgresql-18:latest"
-  #   volumes:
-  #     - "/var/lib/pgsql/data"
-  #   env_file:
-  #     - path: "./default.env"
-  #       required: true
-  #     - path: "./.env"
-  #       required: false
-  #   environment:
-  #     - POSTGRESQL_ADMIN_PASSWORD=${POSTGRES_PASSWORD}
-  #   healthcheck:
-  #     test: ["CMD", "pg_isready", "-U", "postgres"]
-  #     interval: 5s
-  #     timeout: 5s
-  #     retries: 5
+  # Optional PostgreSQL: merge compose-with-db.yaml (see docs/rhdh-local-guide/postgresql-guide.md)
+  # Example: podman compose -f compose.yaml -f compose-with-db.yaml up -d
 
   rhdh:
     container_name: rhdh
@@ -50,8 +31,6 @@ services:
     depends_on:
       install-dynamic-plugins:
         condition: service_completed_successfully
-      # db:
-      #   condition: service_healthy
 
   install-dynamic-plugins:
     container_name: rhdh-plugins-installer

default.env

@@ -4,6 +4,8 @@ POSTGRES_HOST=db
 POSTGRES_PORT=5432
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
+# To pin the image used by compose-with-db.yaml, set POSTGRES_IMAGE in .env (or export it), e.g.:
+# POSTGRES_IMAGE=registry.redhat.io/rhel10/postgresql-18:latest
 
 BASE_URL=http://localhost:7007
 

docs/rhdh-local-guide/postgresql-guide.md

@@ -7,56 +7,37 @@ If you want to use PostgreSQL with RHDH, here are the steps:
 
 The examples below use `podman` and `podman compose`. If you use Docker, replace `podman` with `docker` (for example `docker login`, `docker compose`, `docker exec`).
 
-`default.env` already supplies the `POSTGRES_*` defaults via `env_file`. Put only the values you want to change in your project `.env` (or export them). You do not need to copy every `POSTGRES_*` key.
+`default.env` already supplies the `POSTGRES_*` defaults via `env_file`. Put only the values you want to change in your project `.env` (or export them). You do not need to copy every `POSTGRES_*` key. You can pin the Postgres image with `POSTGRES_IMAGE` in `.env` (see [`compose-with-db.yaml`](https://github.com/redhat-developer/rhdh-local/blob/main/compose-with-db.yaml)).
 
-> **Warning:** If you already run optional Postgres and have a persisted `/var/lib/pgsql/data` volume from an **older major** image, do **not** only change `db.image` to a newer major. Follow [Upgrading PostgreSQL](#upgrading-postgresql) first so the volume is upgraded safely.
+> **Warning:** If you already run optional Postgres and have a persisted `/var/lib/pgsql/data` volume from an **older major** image, do **not** only bump `POSTGRES_IMAGE` (or the default image major). Follow [Upgrading PostgreSQL](#upgrading-postgresql) first so the volume is upgraded safely.
 
 1. Login to container registry with *Red Hat Login* credentials to use `postgresql` image
 
    ```sh
    podman login registry.redhat.io
    ```
 
-2. Uncomment the `db` service block in [https://github.com/redhat-developer/rhdh-local/blob/main/compose.yaml](https://github.com/redhat-developer/rhdh-local/blob/main/compose.yaml) file
+2. Start RHDH with the optional Postgres overlay [`compose-with-db.yaml`](https://github.com/redhat-developer/rhdh-local/blob/main/compose-with-db.yaml). Note that the order of the YAML files is important:
 
-   ```yaml
-   db:
-     image: "registry.redhat.io/rhel10/postgresql-18:latest"
-     volumes:
-       - "/var/lib/pgsql/data"
-     env_file:
-       - path: "./default.env"
-         required: true
-       - path: "./.env"
-         required: false
-     environment:
-       - POSTGRESQL_ADMIN_PASSWORD=${POSTGRES_PASSWORD}
-     healthcheck:
-       test: ["CMD", "pg_isready", "-U", "postgres"]
-       interval: 5s
-       timeout: 5s
-       retries: 5
+   ```sh
+   podman compose -f compose.yaml -f compose-with-db.yaml up -d
    ```
 
-3. Uncomment the `db` section in the `depends_on` section of `rhdh` service in [https://github.com/redhat-developer/rhdh-local/blob/main/compose.yaml](https://github.com/redhat-developer/rhdh-local/blob/main/compose.yaml)
+   You can combine this with other overlays the same way. For example, with the [corporate proxy](corporate-proxy-setup-sim.md) setup:
 
-   ```yaml
-   depends_on:
-     install-dynamic-plugins:
-       condition: service_completed_successfully
-     db:
-       condition: service_healthy
+   ```sh
+   podman compose -f compose.yaml -f compose-with-db.yaml -f compose-with-corporate-proxy.yaml up -d
    ```
 
-4. Comment out the SQLite in-memory configuration in [`app-config.local.yaml`](https://github.com/redhat-developer/rhdh-local/blob/main/configs/app-config/app-config.local.example.yaml)
+3. Comment out the SQLite in-memory configuration in [`app-config.local.yaml`](https://github.com/redhat-developer/rhdh-local/blob/main/configs/app-config/app-config.local.example.yaml)
 
    ```yaml
    # database:
    #   client: better-sqlite3
    #   connection: ':memory:'
    ```
 
-5. Add Postgres configuration in [`app-config.local.yaml`](https://github.com/redhat-developer/rhdh-local/blob/main/configs/app-config/app-config.local.example.yaml)
+4. Add Postgres configuration in [`app-config.local.yaml`](https://github.com/redhat-developer/rhdh-local/blob/main/configs/app-config/app-config.local.example.yaml)
 
    ```yaml
    database:
@@ -90,6 +71,10 @@ The new image must support upgrading from your current major version (its `POSTG
 
 > **Warning:** Back up the Postgres data volume (or take a host-level snapshot) before upgrading. Stop RHDH first so nothing writes to the database during the upgrade. The `copy` mode needs roughly as much free space as the current data directory.
 
+Do not edit tracked [`compose-with-db.yaml`](https://github.com/redhat-developer/rhdh-local/blob/main/compose-with-db.yaml) for the upgrade. Put temporary settings in gitignored `compose.override.yaml` instead. Pulling a newer default image major in `compose-with-db.yaml` is not a silent safe upgrade for an existing data volume — follow the steps below when the image major changes.
+
+When using `-f compose-with-db.yaml`, Compose does not auto-load `compose.override.yaml`. Include it explicitly on upgrade commands. Keep every other `-f` overlay you already use (for example `-f compose-with-corporate-proxy.yaml`).
+
 The `psql` examples below use `POSTGRES_USER` from the container environment (`default.env` / `.env`). `sh -c` is required so the variable expands inside the container.
 
 ### Steps
@@ -104,27 +89,27 @@ The `psql` examples below use `POSTGRES_USER` from the container environment (`d
 2. Stop RHDH so it does not write during the upgrade:
 
    ```sh
-   podman compose stop rhdh
+   podman compose -f compose.yaml -f compose-with-db.yaml stop rhdh
    ```
 
-3. In `compose.yaml`, set `db.image` to the newer Postgres image and add `POSTGRESQL_UPGRADE=copy` for this boot only:
+3. Point at the target major image and enable a one-time upgrade boot:
 
-   ```yaml
-   db:
-     image: "registry.redhat.io/<newer-postgresql-image>:latest"
-     # ...existing volumes, env_file, healthcheck...
-     environment:
-       - POSTGRESQL_ADMIN_PASSWORD=${POSTGRES_PASSWORD}
-       - POSTGRESQL_UPGRADE=copy
+   - In your project `.env`, set `POSTGRES_IMAGE` to the newer image (for example `registry.redhat.io/rhel10/postgresql-18:latest`).
+   - Copy the temporary override example:
+
+   ```sh
+   cp compose.postgres-upgrade.override.example.yaml compose.override.yaml
    ```
 
+   That override only adds `POSTGRESQL_UPGRADE=copy` for this boot.
+
 4. Recreate and start the `db` **container** so it boots the new image against the **existing** data volume (do **not** run `podman compose down --volumes` / `docker compose down --volumes`):
 
    ```sh
-   podman compose up -d db
+   podman compose -f compose.yaml -f compose-with-db.yaml -f compose.override.yaml up -d db
    ```
 
-   Wait until `db` is healthy (`podman compose ps`), then confirm the new major version:
+   Wait until `db` is healthy (`podman compose -f compose.yaml -f compose-with-db.yaml -f compose.override.yaml ps`), then confirm the new major version:
 
    ```sh
    podman exec db sh -c 'psql -U "${POSTGRES_USER:-postgres}" -c "SHOW server_version;"'
@@ -141,24 +126,28 @@ The `psql` examples below use `POSTGRES_USER` from the container environment (`d
    # podman exec db sh -c 'psql -U "${POSTGRES_USER:-postgres}" -c "ALTER DATABASE \"<dbname>\" REFRESH COLLATION VERSION;"'
    ```
 
-6. Remove `POSTGRESQL_UPGRADE=copy` from `compose.yaml`, then force-recreate only the `db` **container** so the updated environment takes effect:
+6. Remove `POSTGRESQL_UPGRADE` by deleting `compose.override.yaml` (or stripping that env from it), then force-recreate only the `db` **container** so the updated environment takes effect:
 
    ```sh
-   podman compose up -d --force-recreate db
+   rm compose.override.yaml
+   podman compose -f compose.yaml -f compose-with-db.yaml up -d --force-recreate db
    ```
 
+   Keep `POSTGRES_IMAGE` in `.env` if you want to pin the major; otherwise the default from `compose-with-db.yaml` applies.
+
    `--force-recreate` replaces the container; it does **not** create a fresh database or wipe `/var/lib/pgsql/data`. Compose keeps the existing volume as long as you do not pass `--volumes` / `-v` to `podman compose down` / `docker compose down` or otherwise remove that volume.
 
 7. Start RHDH again and verify the instance:
 
    ```sh
-   podman compose up -d rhdh
+   podman compose -f compose.yaml -f compose-with-db.yaml up -d rhdh
    ```
 
    Open [http://localhost:7007](http://localhost:7007) and confirm your catalog (or other persisted data) is still present.
 
 ### What not to do
 
+- Do not edit `compose-with-db.yaml` for upgrades (use `.env` + temporary `compose.override.yaml`).
 - Do not delete the Postgres data volume as part of this upgrade (`podman compose down --volumes` / `docker compose down --volumes`, `volume rm`, pruning volumes, etc.).
 - Do not treat `--force-recreate db` as a data reset — it only recreates the container.
 - Do not leave `POSTGRESQL_UPGRADE` set after the upgrade succeeds.