Production deploy

.github/workflows/branches.yml

@@ -87,6 +87,8 @@ jobs:
           docker tag etnaagent/vulcan_c4_env:latest vulcan_c4_env
       - name: Build etna-base image
         run: |
+          docker build -t development-certs -f development-certs/Dockerfile development-certs
+          docker build -t etna-base-dev -f docker/etna-base-dev/Dockerfile docker/etna-base-dev
           docker build -t etna -f etna/Dockerfile --build-arg APP_NAME=etna etna
           docker build -t etna-base -f docker/etna-base/Dockerfile --build-arg APP_NAME=etna-base docker/etna-base
       - name: Build vulcan_app image

.github/workflows/gem-etna-push.yml

@@ -11,10 +11,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Ruby 2.7
+      - name: Set up Ruby 3.4
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '2.7'
+          ruby-version: '3.4'
           bundler-cache: true
 
       - name: Publish to RubyGems

.github/workflows/rollback-production-image.yml

@@ -0,0 +1,69 @@
+name: Rollback production image tag
+
+on:
+  workflow_dispatch:
+    inputs:
+      image:
+        description: "DockerHub image to retag"
+        required: true
+        type: choice
+        options:
+          - etna
+          - magma
+          - metis
+          - janus
+          - gnomon
+          - polyphemus
+          - vulcan
+          - timur
+          - vesta
+      digest:
+        description: "DockerHub digest to restore, e.g. sha256:..."
+        required: true
+        type: string
+
+jobs:
+  retag-production:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Validate inputs
+        env:
+          IMAGE: ${{ inputs.image }}
+          DIGEST: ${{ inputs.digest }}
+        run: |
+          set -euo pipefail
+
+          case "$IMAGE" in
+            etna|magma|metis|janus|gnomon|polyphemus|vulcan|timur|vesta)
+              ;;
+            *)
+              echo "Unsupported image: $IMAGE" >&2
+              exit 1
+              ;;
+          esac
+
+          if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+            echo "Digest must look like sha256:<64 lowercase hex chars>" >&2
+            exit 1
+          fi
+
+      - name: Docker login
+        env:
+          DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+        run: echo "${DOCKER_TOKEN}" | docker login -u etnaagent --password-stdin
+
+      - name: Move production tag back to digest
+        env:
+          IMAGE: ${{ inputs.image }}
+          DIGEST: ${{ inputs.digest }}
+        run: |
+          set -euo pipefail
+
+          docker buildx imagetools create \
+            -t "etnaagent/${IMAGE}:production" \
+            "etnaagent/${IMAGE}@${DIGEST}"
+
+      - name: Verify production tag
+        env:
+          IMAGE: ${{ inputs.image }}
+        run: docker buildx imagetools inspect "etnaagent/${IMAGE}:production"

Dockerfile.ruby34-test

@@ -0,0 +1,66 @@
+FROM ruby:3.4-bullseye
+WORKDIR /app
+
+ENV RUBY_VERSION 3.4.9
+ENV RUBY_MAJOR_VERSION 3.4.0
+ENV BUNDLER_VERSION_ 2.6.2
+ENV DOCKERIZE_VERSION 0.5.0
+ENV DOCKERIZE_URL https://github.com/jwilder/dockerize/releases/download/v${DOCKERIZE_VERSION}/dockerize-linux-amd64-v${DOCKERIZE_VERSION}.tar.gz
+ENV RACK_ENV development
+
+ENV BUNDLE_PATH="/bundle"
+ENV BUNDLE_BIN="$BUNDLE_PATH/ruby/$RUBY_MAJOR_VERSION/bin"
+ENV BUNDLE_SILENCE_ROOT_WARNING=0
+ENV BUNDLE_APP_CONFIG="/etna/.bundle"
+ENV BUNDLE_GEMFILE="/etna/Gemfile"
+ENV PATH "/app/bin:/etna/node_modules/.bin:/app/node_modules/.bin:$BUNDLE_BIN:$PATH"
+
+RUN apt-get update \
+  && apt-get install -y \
+    libpq-dev \
+    postgresql-client \
+    libvips \
+    libvips-dev \
+    libcurl4-openssl-dev \
+    libssl-dev \
+    rsync \
+    sshpass \
+    sqlite3 \
+    --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN curl -o /tmp/dockerize.tgz -L $DOCKERIZE_URL && ( cd /usr/bin && tar xzf /tmp/dockerize.tgz )
+
+RUN gem install bundler -v "=$BUNDLER_VERSION_"
+
+ENV GEM_HOME="$BUNDLE_PATH"
+ENV GEM_PATH="/root/.gem/ruby/$RUBY_MAJOR_VERSION:/usr/local/lib/ruby/gems/$RUBY_MAJOR_VERSION:/usr/local/bundle:$BUNDLE_PATH/ruby/$RUBY_MAJOR_VERSION"
+
+# Copy entrypoints from the dev base image directory
+COPY docker/etna-base-dev/entrypoints /entrypoints
+
+# Copy only dependency inputs first so Docker can cache native gem builds.
+# The local etna path gem needs its gemspec, lib files, extensions, and completion files.
+COPY etna/Gemfile etna/Gemfile.lock etna/etna.gemspec /etna/
+COPY etna/lib /etna/lib
+COPY etna/ext /etna/ext
+COPY etna/*.completion /etna/
+
+# Install all gems with detailed logging so long-running steps are visible.
+RUN bash -lc 'set -euo pipefail; \
+  echo "[bundle] start: $(date -u +%Y-%m-%dT%H:%M:%SZ)"; \
+  echo "[bundle] ruby: $(ruby -v)"; \
+  echo "[bundle] bundler: $(bundle -v)"; \
+  echo "[bundle] gem source(s):"; \
+  bundle config get source || true; \
+  echo "[bundle] lock platforms:"; \
+  ruby -e "lock = File.read(\"/etna/Gemfile.lock\"); puts(lock[/PLATFORMS\\n(.*?)\\n\\n/m, 1] || \"(missing)\")"; \
+  echo "[bundle] install begin"; \
+  time bundle install -j 6 --verbose; \
+  echo "[bundle] install done: $(date -u +%Y-%m-%dT%H:%M:%SZ)"'
+
+# Copy the rest of the etna source after bundle install so app code changes do not
+# force recompilation of native gems like curb.
+COPY etna /etna
+
+RUN mkdir -p /app/tmp/pids /app/public/js /app/public/css /app/log /app/data /tmp/metrics.prom

docker/etna-base-dev/Dockerfile

@@ -1,14 +1,12 @@
 FROM development-certs
 
-FROM ruby:2.7.6-bullseye
+FROM ruby:3.4-bullseye
 WORKDIR /app
 
 ENV DOCKER_VERSION 20.10.6
-ENV RUBY_VERSION 2.7.6
-ENV RUBY_MAJOR_VERSION 2.7.0
-# This is so dumb -- but if BUNDLER_VERSION is set as an environment variable,
-# it causes issues with bundler... bleck.  So, so add an underscore at the end.
-ENV BUNDLER_VERSION_ 2.3.15
+ENV RUBY_VERSION 3.4.9
+ENV RUBY_MAJOR_VERSION 3.4.0
+ENV BUNDLER_VERSION_ 2.6.2
 ENV DOCKERIZE_VERSION 0.5.0
 ENV NODE_VERSION 14.17.1
 ENV DOCKERIZE_URL https://github.com/jwilder/dockerize/releases/download/v${DOCKERIZE_VERSION}/dockerize-linux-amd64-v${DOCKERIZE_VERSION}.tar.gz
@@ -24,7 +22,7 @@ ENV PATH "/app/bin:/etna/node_modules/.bin:/app/node_modules/.bin:$BUNDLE_BIN:$P
 COPY --from=0 /certs/rootCA.pem /usr/local/share/ca-certificates/devRoot.crt
 RUN chmod 644 /usr/local/share/ca-certificates/devRoot.crt && update-ca-certificates
 
-# add postgres 10 repo for debian bionic
+# add postgres 14 repo for debian bullseye
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ bullseye-pgdg main" >> /etc/apt/sources.list.d/pgdg.list
 RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 
@@ -49,10 +47,7 @@ RUN curl https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x6
 
 RUN curl -o /tmp/dockerize.tgz -L $DOCKERIZE_URL && ( cd /usr/bin && tar xzf /tmp/dockerize.tgz )
 
-RUN rm /usr/local/lib/ruby/gems/2.7.0/specifications/default/bundler-*.gemspec
 RUN gem install bundler --default -v "=$BUNDLER_VERSION_"
-## !! Note the rubygems version locked down here is meant to just be a temporary fix and should be removed when we do update Ruby !!
-RUN gem update --system 3.4.22
 
 # We also the GEM_PATH to be this BUNDLE_PATH so that ides (Rubymine) will find gems easier.
 ENV GEM_HOME="$BUNDLE_PATH"

etna/.github/workflows/gem-push.yml

@@ -11,10 +11,11 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby 2.5
-      uses: actions/setup-ruby@v1
+    - name: Set up Ruby 3.4
+      uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 2.5.x
+        ruby-version: '3.4'
+        bundler-cache: true
 
     - name: Publish to RubyGems
       run: |

etna/.github/workflows/ruby-tests.yml

@@ -5,22 +5,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/cache@v2
+    - name: Set up Ruby 3.4
+      uses: ruby/setup-ruby@v1
       with:
-        path: |
-          vendor/bundle
-        key: ${{ runner.os }}-gem-${{ hashFiles('**/Gemfile.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-gem
-    - name: Set up Ruby
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.5
-    - name: Bundle install Gems
-      run: |
-        gem install bundler -v 2.2.0
-        bundle config path vendor/bundle
-        bundle install --jobs 4 --retry 3
+        ruby-version: '3.4'
+        bundler-cache: true
     - name: Run test suite
       run: |
         bundle exec rspec spec/

etna/Gemfile

@@ -1,13 +1,14 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
-ruby "~> 2.7"
+ruby "~> 3.4"
 
 gem 'mini_magick'
 gem 'daru'
+gem 'matrix' # Ruby 3.4: daru requires matrix, which is no longer bundled by default.
 gem 'rltk'
 gem 'filigree', '0.3.3'
 gem 'fog-aws'
-gem 'puma', '>=5.0.2'
+gem 'puma', '~> 6.4' # Ruby 3.4: Puma 5.x is not compatible with Ruby 3.4.
 gem 'etna', path: '/etna'
 gem 'pg'
 gem 'sequel'
@@ -29,8 +30,8 @@ gem 'activesupport', '>= 4.2.6'
 gem 'addressable', '~> 2.8.1'
 gem "yabeda"
 gem "yabeda-prometheus"
-gem "yabeda-puma-plugin"
-gem "curb"
+gem "yabeda-puma-plugin", "~> 0.7", ">= 0.7.1" # Puma 6: 0.6.0 calls APIs removed in Puma 6.
+gem "curb", "1.0.8" # Ruby 3.4 amd64: newer curb native builds hung under Docker emulation.
 gem "net-ssh"
 gem "net-scp"
 gem "net-sftp"
@@ -39,17 +40,22 @@ gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0'
 gem 'connection_pool'
 gem "rsync", "~> 1.0", ">= 1.0.9"
 gem "json_schemer"
+gem 'base64' # Ruby 3.4: stdlib gem used directly by etna client/token code.
+gem 'csv' # Ruby 3.4: stdlib gem used directly by etna CSV helpers/workflows.
+gem 'logger' # Ruby 3.4: stdlib gem used directly by Etna::Logger and commands.
+gem 'net-smtp' # Ruby 3.4: stdlib gem required by Etna::Controller mail handling.
+gem 'ostruct' # Ruby 3.4/3.5: stdlib gem used by etna clients and filigree.
 
 group :test do
   gem 'rspec'
   gem 'rack-test', require: "rack/test"
-  gem "pry", "~>0.13.0"
+  gem "pry", "~> 0.15.2" # Ruby 3.4: pry 0.13.x uses APIs removed from Ruby.
   gem "pry-byebug"
   gem "timecop"
   gem "webmock"
-  gem "debase"
   gem "simplecov"
   gem 'factory_bot'
-  gem 'database_cleaner', '1.8.5'
+  gem 'database_cleaner'
+  gem 'database_cleaner-sequel' # database_cleaner 2.x: Sequel adapter is split into its own gem.
   gem 'net-http-persistent'
 end