From 12b997abcc7cd2980b6dadf2f11fe757a7b75971 Mon Sep 17 00:00:00 2001 From: Giuseppe Carboni Date: Fri, 20 Mar 2026 16:03:25 +0100 Subject: [PATCH] Updated procedure in order to publish on GHCR GHCR = GitHub Container Registry The procedure has now improved so that each container build layer is less than 10GB, so that it can be uploaded on GHCR safely. Each image is tagged with - and uploaded separately. Additionaly, the resulting image is also published as latest so that the latest tag always points to the latest image built. --- .github/utils/download_from_gdrive.py | 39 ---- .github/utils/gdrive_requirements.txt | 4 - .github/utils/upload_to_gdrive.py | 46 ---- .github/workflows/provision.yml | 30 ++- ansible/Medicina.yml | 14 +- ansible/Noto.yml | 14 +- ansible/SRT.yml | 14 +- ansible/development.yml | 62 +++++- ansible/roles/acs/tasks/java.yml | 30 --- ansible/roles/acs/tasks/main.yml | 29 --- ansible/roles/acs/tasks/receivers_monitor.yml | 21 -- ansible/roles/acs/tasks/utilities.yml | 43 ---- .../tasks/main.yml} | 8 - .../templates/_discos-check-branch | 0 .../{acs => acs_environment}/templates/bashrc | 2 +- .../templates/dir_colors | 0 .../templates/discos-get | 0 .../templates/discos-set | 0 .../templates/load_branch | 0 .../templates/prompt_command.sh | 0 .../{acs => acs_environment}/templates/pyrc | 0 .../{acs => acs_environment}/templates/vimrc | 0 .../{acs => acs_install}/defaults/main.yml | 1 - .../acs.yml => acs_install/tasks/main.yml} | 20 -- .../tasks/patches/ACS-2021DEC.yml | 0 .../templates/acsStartContainerGdb | 0 .../patches/ACS-2021DEC/BaciHelper.py | 0 .../patches/ACS-2021DEC/ExecuteContainer.java | 0 .../patches/ACS-2021DEC/Executor.java | 0 .../patches/ACS-2021DEC/WDALImpl.java | 0 .../templates/patches/ACS-2021DEC/buildDDS | 0 .../patches/ACS-2021DEC/buildOmniORB | 0 .../patches/ACS-2021DEC/buildPyModules | 0 .../templates/patches/ACS-2021DEC/buildPython | 0 .../patches/ACS-2021DEC/enumpropAlarm.i | 0 ansible/roles/acs_prepare/defaults/main.yml | 3 + .../yum.yml => acs_prepare/tasks/main.yml} | 60 ++++- .../discos_dependencies/defaults/main.yml | 24 ++ .../tasks/main.yml} | 48 ++++ deployment/__init__.py | 16 +- scripts/discos-container | 205 +++++++++++++----- 41 files changed, 418 insertions(+), 315 deletions(-) delete mode 100755 .github/utils/download_from_gdrive.py delete mode 100644 .github/utils/gdrive_requirements.txt delete mode 100755 .github/utils/upload_to_gdrive.py delete mode 100644 ansible/roles/acs/tasks/java.yml delete mode 100644 ansible/roles/acs/tasks/main.yml delete mode 100644 ansible/roles/acs/tasks/receivers_monitor.yml delete mode 100644 ansible/roles/acs/tasks/utilities.yml rename ansible/roles/{acs/tasks/discos_environment.yml => acs_environment/tasks/main.yml} (95%) rename ansible/roles/{acs => acs_environment}/templates/_discos-check-branch (100%) rename ansible/roles/{acs => acs_environment}/templates/bashrc (98%) rename ansible/roles/{acs => acs_environment}/templates/dir_colors (100%) rename ansible/roles/{acs => acs_environment}/templates/discos-get (100%) rename ansible/roles/{acs => acs_environment}/templates/discos-set (100%) rename ansible/roles/{acs => acs_environment}/templates/load_branch (100%) rename ansible/roles/{acs => acs_environment}/templates/prompt_command.sh (100%) rename ansible/roles/{acs => acs_environment}/templates/pyrc (100%) rename ansible/roles/{acs => acs_environment}/templates/vimrc (100%) rename ansible/roles/{acs => acs_install}/defaults/main.yml (98%) rename ansible/roles/{acs/tasks/acs.yml => acs_install/tasks/main.yml} (93%) rename ansible/roles/{acs => acs_install}/tasks/patches/ACS-2021DEC.yml (100%) rename ansible/roles/{acs => acs_install}/templates/acsStartContainerGdb (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/BaciHelper.py (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/ExecuteContainer.java (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/Executor.java (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/WDALImpl.java (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/buildDDS (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/buildOmniORB (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/buildPyModules (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/buildPython (100%) rename ansible/roles/{acs => acs_install}/templates/patches/ACS-2021DEC/enumpropAlarm.i (100%) create mode 100644 ansible/roles/acs_prepare/defaults/main.yml rename ansible/roles/{acs/tasks/yum.yml => acs_prepare/tasks/main.yml} (70%) create mode 100644 ansible/roles/discos_dependencies/defaults/main.yml rename ansible/roles/{acs/tasks/discos_dependencies.yml => discos_dependencies/tasks/main.yml} (95%) diff --git a/.github/utils/download_from_gdrive.py b/.github/utils/download_from_gdrive.py deleted file mode 100755 index 1f5c74f6..00000000 --- a/.github/utils/download_from_gdrive.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -from google.oauth2.credentials import Credentials -from google.auth.transport.requests import Request -from googleapiclient.discovery import build -from googleapiclient.http import MediaIoBaseDownload -import os -import io - -TOKEN_FILE = 'token.json' -CONTAINER_FILE_PATH = '/home/runner/discos_manager.tar' -SCOPES = ['https://www.googleapis.com/auth/drive.readonly'] - -# Create the token file from the GH Secret -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(os.environ.get('GOOGLE_DRIVE_TOKEN')) - -# Authenticate with the token and eventually update it -creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES) -if creds.expired and creds.refresh_token: - creds.refresh(Request()) - -service = build('drive', 'v3', credentials=creds) - -downloader = MediaIoBaseDownload( - io.FileIO(VM_FILE_PATH, 'wb'), - service.files().get_media( - fileId=os.environ.get('PROVISIONED_CONTAINER_GDRIVE_ID'), - ), - chunksize=64*1024*1024 -) -done = False -while not done: - status, done = downloader.next_chunk() - if status is not None: - print(f'Download progress: {int(status.progress() * 100)}%', flush=True) - -# Finally update the token file -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(creds.to_json()) diff --git a/.github/utils/gdrive_requirements.txt b/.github/utils/gdrive_requirements.txt deleted file mode 100644 index 7c686617..00000000 --- a/.github/utils/gdrive_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -google-auth -google-auth-oauthlib -google-auth-httplib2 -google-api-python-client diff --git a/.github/utils/upload_to_gdrive.py b/.github/utils/upload_to_gdrive.py deleted file mode 100755 index 67c818fe..00000000 --- a/.github/utils/upload_to_gdrive.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -from google.oauth2.credentials import Credentials -from google.auth.transport.requests import Request -from googleapiclient.discovery import build -from googleapiclient.http import MediaFileUpload -import os - -TOKEN_FILE = 'token.json' -CONTAINER_FILE_PATH = '/home/runner/discos_manager.tar' -SCOPES = [ - 'https://www.googleapis.com/auth/drive', - 'https://www.googleapis.com/auth/drive.file' -] - -# Create the token file from the GH Secret -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(os.environ.get('GOOGLE_DRIVE_TOKEN')) - -# Authenticate with the token and eventually update it -creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES) -if creds.expired and creds.refresh_token: - creds.refresh(Request()) - -# Prepare the files to be uploaded -service = build('drive', 'v3', credentials=creds) -container_media = MediaFileUpload( - CONTAINER_FILE_PATH, - mimetype='application/x-tar', - resumable=True, - chunksize=64*1024*1024 -) -request = service.files().update( - fileId=os.environ.get('PROVISIONED_CONTAINER_GDRIVE_ID'), - media_body=container_media, - fields='id' -) - -response = None -while response is None: - status, response = request.next_chunk() - if status is not None: - print(f'Upload progress: {int(status.progress() * 100)}%', flush=True) - -# Finally update the token file -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(creds.to_json()) diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml index 46dc3994..1e7bc319 100644 --- a/.github/workflows/provision.yml +++ b/.github/workflows/provision.yml @@ -9,14 +9,15 @@ on: schedule: - cron: '0 6 * * 1' +permissions: + contents: read + packages: write + jobs: provision: + runs-on: ubuntu-latest env: REPOSITORY_TOKEN: "${{ secrets.DEPENDENCIES_TOKEN }}" - GH_TOKEN: "${{ secrets.GH_WORKFLOWS_TOKEN }}" - GOOGLE_DRIVE_TOKEN: "${{ secrets.GOOGLE_DRIVE_TOKEN }}" - PROVISIONED_CONTAINER_GDRIVE_ID: "${{ secrets.PROVISIONED_CONTAINER_GDRIVE_ID }}" - runs-on: ubuntu-latest steps: - name: Free up space uses: jlumbroso/free-disk-space@main @@ -44,16 +45,21 @@ jobs: else discos-deploy manager:development --docker --default-passwords fi - - name: Save provisioned image as tar artifact + - name: Login to GHCR + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Tag and push images to GHCR + if: github.event_name == 'push' && github.ref == 'refs/heads/master' run: | - discos-container save discos_manager.tar - working-directory: /home/runner + IMAGE_TAG="${GITHUB_REF_NAME}-${GITHUB_SHA::7}" + discos-container tag latest "${IMAGE_TAG}" + discos-container push --tag latest + discos-container push --tag "${IMAGE_TAG}" - name: Shutdown the container if: always() run: | discos-container stop || true - - name: Upload the Docker image to Google Drive - run: | - pip install -r .github/utils/gdrive_requirements.txt - python .github/utils/upload_to_gdrive.py - gh secret set GOOGLE_DRIVE_TOKEN --org discos --visibility selected --repos discos,deployment < token.json diff --git a/ansible/Medicina.yml b/ansible/Medicina.yml index f2244d40..7a2abef6 100644 --- a/ansible/Medicina.yml +++ b/ansible/Medicina.yml @@ -120,12 +120,22 @@ when: lustre_server_ip is defined -- name: Apply ACS configuration to ACS nodes +- name: Prepare for ACS setup tags: provision hosts: acs_meta remote_user: root roles: - - acs + - acs_prepare + - acs_environment + + +- name: Install ACS + tags: provision + hosts: acs_meta + remote_user: root + roles: + - acs_install + - discos_dependencies - name: Configure the manager machine diff --git a/ansible/Noto.yml b/ansible/Noto.yml index 85f4a175..766abe40 100644 --- a/ansible/Noto.yml +++ b/ansible/Noto.yml @@ -97,12 +97,22 @@ loop_var: username -- name: Apply ACS configuration to ACS nodes +- name: Prepare for ACS setup tags: provision hosts: manager remote_user: root roles: - - acs + - acs_prepare + - acs_environment + + +- name: Install ACS + tags: provision + hosts: manager + remote_user: root + roles: + - acs_install + - discos_dependencies - name: Configure the manager machine diff --git a/ansible/SRT.yml b/ansible/SRT.yml index ce339cf4..303a24fc 100644 --- a/ansible/SRT.yml +++ b/ansible/SRT.yml @@ -123,12 +123,22 @@ - sardara_remote_directory is defined -- name: Apply ACS configuration to ACS nodes +- name: Prepare for ACS setup tags: provision hosts: acs_meta remote_user: root roles: - - acs + - acs_prepare + - acs_environment + + +- name: Install ACS + tags: provision + hosts: acs_meta + remote_user: root + roles: + - acs_install + - discos_dependencies - name: Configure the manager machine diff --git a/ansible/development.yml b/ansible/development.yml index 9fa0ab03..49a04b98 100644 --- a/ansible/development.yml +++ b/ansible/development.yml @@ -111,12 +111,58 @@ loop_var: username -- name: Apply ACS configuration to ACS nodes +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" + + +- name: Prepare for ACS setup + tags: provision + hosts: manager + remote_user: root + roles: + - acs_prepare + - acs_environment + + +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" + + +- name: Install ACS tags: provision hosts: manager remote_user: root roles: - - acs + - acs_install + - discos_dependencies + + +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" - name: Configure the manager machine @@ -183,6 +229,18 @@ - vnc +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" + + - name: Deploy DISCOS tags: deploy hosts: manager diff --git a/ansible/roles/acs/tasks/java.yml b/ansible/roles/acs/tasks/java.yml deleted file mode 100644 index be28feaf..00000000 --- a/ansible/roles/acs/tasks/java.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -- name: Install correct java yum packages - ansible.builtin.yum: - name: - - java-11-openjdk - - java-11-openjdk-headless - - java-11-openjdk-devel - state: present - - -- name: Fetch the installed yum packages - ansible.builtin.yum: - list: installed - register: package_facts - - -- name: Filter OpenJDK packages to be removed (non-Java 11) - ansible.builtin.set_fact: - openjdk_packages_to_remove: "{{ package_facts.results | - selectattr('name', 'match', '^java-[^-]+-openjdk(-(headless|devel))?$') | - selectattr('name', 'search', '^(?!java-11-openjdk)') | - map(attribute='name') | list }}" - - -- name: Uninstall OpenJDK packages different from Java 11 - ansible.builtin.yum: - name: "{{ item }}" - state: absent - with_items: "{{ openjdk_packages_to_remove }}" diff --git a/ansible/roles/acs/tasks/main.yml b/ansible/roles/acs/tasks/main.yml deleted file mode 100644 index faf894e1..00000000 --- a/ansible/roles/acs/tasks/main.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- - -- name: Install ACS yum packages - ansible.builtin.include_tasks: yum.yml - - -- name: Install correct JAVA packages - ansible.builtin.include_tasks: java.yml - - -- name: Configure the DISCOS environment - ansible.builtin.include_tasks: discos_environment.yml - - -- name: Install ACS - ansible.builtin.include_tasks: acs.yml - - -- name: Install the DISCOS dependencies - ansible.builtin.include_tasks: discos_dependencies.yml - - -- name: Install some utilities - ansible.builtin.include_tasks: utilities.yml - - -- name: Configure the monitoring for the receivers - ansible.builtin.include_tasks: receivers_monitor.yml - when: monitor_receivers diff --git a/ansible/roles/acs/tasks/receivers_monitor.yml b/ansible/roles/acs/tasks/receivers_monitor.yml deleted file mode 100644 index 480f4f08..00000000 --- a/ansible/roles/acs/tasks/receivers_monitor.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- - -- name: Create the /service/receivers directory - ansible.builtin.file: - path: "/service/receivers" - state: directory - owner: "{{ user.name }}" - group: "{{ user.group }}" - mode: "0755" - recurse: true - - -- name: Add receivers monitoring script to rc.local - ansible.builtin.blockinfile: - path: "/etc/rc.local" - state: present - marker: "######## Receivers monitoring {mark} ########" - block: | - if [ -f /{{ discos_sw_dir }}/introots/default/bin/receiversmonitor.py ]; then - runuser -l {{ user.name }} -c 'receiversmonitor.py &' - fi diff --git a/ansible/roles/acs/tasks/utilities.yml b/ansible/roles/acs/tasks/utilities.yml deleted file mode 100644 index cdf75058..00000000 --- a/ansible/roles/acs/tasks/utilities.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- - -- name: Install prospector (Python linter) - ansible.builtin.command: "pip install prospector" - become: true - become_user: "{{ user.name }}" - become_flags: "-i" - changed_when: true - - -- name: Install ipython - ansible.builtin.command: "pip install ipython" - become: true - become_user: "{{ user.name }}" - become_flags: "-i" - changed_when: true - - -- name: Create default ipython profile - ansible.builtin.command: "ipython profile create" - become: true - become_user: "{{ item.value.name }}" - become_flags: "-i" - with_dict: "{{ users }}" - changed_when: true - - -- name: Set the default application for html files - ansible.builtin.lineinfile: - path: "/usr/share/applications/mimeapps.list" - line: "{{ item }}" - state: present - create: true - mode: "0644" - with_items: - - "[Default Applications]" - - "text/html=firefox.desktop" - - -- name: Allow all users to have access to Python executables - ansible.builtin.lineinfile: - dest: /etc/bashrc - line: 'export PATH=/alma/{{ acs_tag }}/pyenv/shims:$PATH' diff --git a/ansible/roles/acs/tasks/discos_environment.yml b/ansible/roles/acs_environment/tasks/main.yml similarity index 95% rename from ansible/roles/acs/tasks/discos_environment.yml rename to ansible/roles/acs_environment/tasks/main.yml index 9cbfbfae..1c039db7 100644 --- a/ansible/roles/acs/tasks/discos_environment.yml +++ b/ansible/roles/acs_environment/tasks/main.yml @@ -158,11 +158,3 @@ fi no_log: true with_dict: "{{ users }}" - - -- name: Source scl_source - ansible.builtin.lineinfile: - dest: "{{ item }}" - line: 'source scl_source enable devtoolset-{{ devtoolset_release }}' - with_items: - - "/etc/bashrc" diff --git a/ansible/roles/acs/templates/_discos-check-branch b/ansible/roles/acs_environment/templates/_discos-check-branch similarity index 100% rename from ansible/roles/acs/templates/_discos-check-branch rename to ansible/roles/acs_environment/templates/_discos-check-branch diff --git a/ansible/roles/acs/templates/bashrc b/ansible/roles/acs_environment/templates/bashrc similarity index 98% rename from ansible/roles/acs/templates/bashrc rename to ansible/roles/acs_environment/templates/bashrc index 97f0f4f6..0da2586a 100644 --- a/ansible/roles/acs/templates/bashrc +++ b/ansible/roles/acs_environment/templates/bashrc @@ -54,7 +54,7 @@ make() { # Variables to export # =================== -export QTDIR={{ qt_dir }} +export QTDIR=/usr/lib64/qt4 export QTLIB=$QTDIR/lib export QTINC=/usr/include/Qt export qt_prefix=$QTLIB diff --git a/ansible/roles/acs/templates/dir_colors b/ansible/roles/acs_environment/templates/dir_colors similarity index 100% rename from ansible/roles/acs/templates/dir_colors rename to ansible/roles/acs_environment/templates/dir_colors diff --git a/ansible/roles/acs/templates/discos-get b/ansible/roles/acs_environment/templates/discos-get similarity index 100% rename from ansible/roles/acs/templates/discos-get rename to ansible/roles/acs_environment/templates/discos-get diff --git a/ansible/roles/acs/templates/discos-set b/ansible/roles/acs_environment/templates/discos-set similarity index 100% rename from ansible/roles/acs/templates/discos-set rename to ansible/roles/acs_environment/templates/discos-set diff --git a/ansible/roles/acs/templates/load_branch b/ansible/roles/acs_environment/templates/load_branch similarity index 100% rename from ansible/roles/acs/templates/load_branch rename to ansible/roles/acs_environment/templates/load_branch diff --git a/ansible/roles/acs/templates/prompt_command.sh b/ansible/roles/acs_environment/templates/prompt_command.sh similarity index 100% rename from ansible/roles/acs/templates/prompt_command.sh rename to ansible/roles/acs_environment/templates/prompt_command.sh diff --git a/ansible/roles/acs/templates/pyrc b/ansible/roles/acs_environment/templates/pyrc similarity index 100% rename from ansible/roles/acs/templates/pyrc rename to ansible/roles/acs_environment/templates/pyrc diff --git a/ansible/roles/acs/templates/vimrc b/ansible/roles/acs_environment/templates/vimrc similarity index 100% rename from ansible/roles/acs/templates/vimrc rename to ansible/roles/acs_environment/templates/vimrc diff --git a/ansible/roles/acs/defaults/main.yml b/ansible/roles/acs_install/defaults/main.yml similarity index 98% rename from ansible/roles/acs/defaults/main.yml rename to ansible/roles/acs_install/defaults/main.yml index ece253a4..015e4071 100644 --- a/ansible/roles/acs/defaults/main.yml +++ b/ansible/roles/acs_install/defaults/main.yml @@ -49,4 +49,3 @@ slalibrary_file: "SlaLibrary.tar.gz" ################ monitor_receivers: false -devtoolset_release: 8 diff --git a/ansible/roles/acs/tasks/acs.yml b/ansible/roles/acs_install/tasks/main.yml similarity index 93% rename from ansible/roles/acs/tasks/acs.yml rename to ansible/roles/acs_install/tasks/main.yml index c631d4db..c072bc19 100644 --- a/ansible/roles/acs/tasks/acs.yml +++ b/ansible/roles/acs_install/tasks/main.yml @@ -1,25 +1,5 @@ --- -- name: Check in which path Java is installed - ansible.builtin.shell: "readlink -f $(which java)" - register: java_home - changed_when: false - - -- name: Make sure /usr/java directory exists - ansible.builtin.file: - path: "/usr/java" - state: directory - mode: "0755" - - -- name: Create a symlink to the correct Java home path, needed by ACS - ansible.builtin.file: - src: "/usr/lib/jvm/java-openjdk" - dest: "/usr/java/default" - state: link - - - name: Create the /alma directory ansible.builtin.file: path: "/alma" diff --git a/ansible/roles/acs/tasks/patches/ACS-2021DEC.yml b/ansible/roles/acs_install/tasks/patches/ACS-2021DEC.yml similarity index 100% rename from ansible/roles/acs/tasks/patches/ACS-2021DEC.yml rename to ansible/roles/acs_install/tasks/patches/ACS-2021DEC.yml diff --git a/ansible/roles/acs/templates/acsStartContainerGdb b/ansible/roles/acs_install/templates/acsStartContainerGdb similarity index 100% rename from ansible/roles/acs/templates/acsStartContainerGdb rename to ansible/roles/acs_install/templates/acsStartContainerGdb diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/BaciHelper.py b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/BaciHelper.py similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/BaciHelper.py rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/BaciHelper.py diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/ExecuteContainer.java b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/ExecuteContainer.java similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/ExecuteContainer.java rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/ExecuteContainer.java diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/Executor.java b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/Executor.java similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/Executor.java rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/Executor.java diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/WDALImpl.java b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/WDALImpl.java similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/WDALImpl.java rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/WDALImpl.java diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildDDS b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildDDS similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildDDS rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildDDS diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildOmniORB b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildOmniORB similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildOmniORB rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildOmniORB diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildPyModules b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPyModules similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildPyModules rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPyModules diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildPython b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPython similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildPython rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPython diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/enumpropAlarm.i b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/enumpropAlarm.i similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/enumpropAlarm.i rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/enumpropAlarm.i diff --git a/ansible/roles/acs_prepare/defaults/main.yml b/ansible/roles/acs_prepare/defaults/main.yml new file mode 100644 index 00000000..352cd177 --- /dev/null +++ b/ansible/roles/acs_prepare/defaults/main.yml @@ -0,0 +1,3 @@ +--- + +devtoolset_release: 8 diff --git a/ansible/roles/acs/tasks/yum.yml b/ansible/roles/acs_prepare/tasks/main.yml similarity index 70% rename from ansible/roles/acs/tasks/yum.yml rename to ansible/roles/acs_prepare/tasks/main.yml index 4e439bf7..5519106d 100644 --- a/ansible/roles/acs/tasks/yum.yml +++ b/ansible/roles/acs_prepare/tasks/main.yml @@ -115,10 +115,18 @@ - name: Disable the devtoolset-{{ devtoolset_release }} sudo command ansible.builtin.file: - path: "/opt/rh/devtoolset-8/root/usr/bin/sudo" + path: "/opt/rh/devtoolset-{{ devtoolset_release }}/root/usr/bin/sudo" mode: "0444" +- name: Source scl_source + ansible.builtin.lineinfile: + dest: "{{ item }}" + line: 'source scl_source enable devtoolset-{{ devtoolset_release }}' + with_items: + - "/etc/bashrc" + + - name: Make sure the default git yum package is not installed ansible.builtin.yum: name: @@ -171,3 +179,53 @@ src: /usr/bin/cmake3 dest: /usr/bin/cmake state: link + + +- name: Install correct java yum packages + ansible.builtin.yum: + name: + - java-11-openjdk + - java-11-openjdk-headless + - java-11-openjdk-devel + state: present + + +- name: Fetch the installed yum packages + ansible.builtin.yum: + list: installed + register: package_facts + + +- name: Filter OpenJDK packages to be removed (non-Java 11) + ansible.builtin.set_fact: + openjdk_packages_to_remove: "{{ package_facts.results | + selectattr('name', 'match', '^java-[^-]+-openjdk(-(headless|devel))?$') | + selectattr('name', 'search', '^(?!java-11-openjdk)') | + map(attribute='name') | list }}" + + +- name: Uninstall OpenJDK packages different from Java 11 + ansible.builtin.yum: + name: "{{ item }}" + state: absent + with_items: "{{ openjdk_packages_to_remove }}" + + +- name: Check in which path Java is installed + ansible.builtin.shell: "readlink -f $(which java)" + register: java_home + changed_when: false + + +- name: Make sure /usr/java directory exists + ansible.builtin.file: + path: "/usr/java" + state: directory + mode: "0755" + + +- name: Create a symlink to the correct Java home path, needed by ACS + ansible.builtin.file: + src: "/usr/lib/jvm/java-openjdk" + dest: "/usr/java/default" + state: link diff --git a/ansible/roles/discos_dependencies/defaults/main.yml b/ansible/roles/discos_dependencies/defaults/main.yml new file mode 100644 index 00000000..fc9e3238 --- /dev/null +++ b/ansible/roles/discos_dependencies/defaults/main.yml @@ -0,0 +1,24 @@ +--- + +cfitsio: { file: 'cfitsio3370.tar.gz', build_dir: 'cfitsio' } +ccfits: { file: 'CCfits-2.4.tar.gz', build_dir: 'CCfits' } +fv: { file: 'fv5.4_pc_linux64.tar.gz' } +modbus: { file: 'libmodbus-3.0.6.tar.gz', build_dir: 'libmodbus-3.0.6' } +f2c: { file: 'f2c.zip'} + +googletest_repo: "447d58b4ee8ea96b4757a5bb5f0b3be75af6c2a1" +googletest_build_dir: "googletest-{{ googletest_repo }}" +googletest_file: "{{ googletest_build_dir }}.zip" +gmock_build: "{{ remote_build_path }}/{{ googletest_build_dir }}/googlemock" +gtest_build: "{{ remote_build_path }}/{{ googletest_build_dir }}/googletest" + +sip_build_dir: sip-4.19.2 +sip_file: "{{ sip_build_dir }}.tar.gz" + +pyqt_build_dir: PyQt4_gpl_x11-4.12 +pyqt_file: "{{ pyqt_build_dir }}.tar.gz" + +pyqtqwt_build_dir: PyQt-Qwt +pyqtqwt_file: "{{ pyqtqwt_build_dir }}.zip" + +slalibrary_file: "SlaLibrary.tar.gz" diff --git a/ansible/roles/acs/tasks/discos_dependencies.yml b/ansible/roles/discos_dependencies/tasks/main.yml similarity index 95% rename from ansible/roles/acs/tasks/discos_dependencies.yml rename to ansible/roles/discos_dependencies/tasks/main.yml index 2c17f90a..38e874a1 100644 --- a/ansible/roles/acs/tasks/discos_dependencies.yml +++ b/ansible/roles/discos_dependencies/tasks/main.yml @@ -960,3 +960,51 @@ become_user: "{{ user.name }}" become_flags: "-i" changed_when: true + + +########### +# Utilities +########### + + +- name: Install prospector (Python linter) + ansible.builtin.command: "pip install prospector" + become: true + become_user: "{{ user.name }}" + become_flags: "-i" + changed_when: true + + +- name: Install ipython + ansible.builtin.command: "pip install ipython" + become: true + become_user: "{{ user.name }}" + become_flags: "-i" + changed_when: true + + +- name: Create default ipython profile + ansible.builtin.command: "ipython profile create" + become: true + become_user: "{{ item.value.name }}" + become_flags: "-i" + with_dict: "{{ users }}" + changed_when: true + + +- name: Set the default application for html files + ansible.builtin.lineinfile: + path: "/usr/share/applications/mimeapps.list" + line: "{{ item }}" + state: present + create: true + mode: "0644" + with_items: + - "[Default Applications]" + - "text/html=firefox.desktop" + + +- name: Allow all users to have access to Python executables + ansible.builtin.lineinfile: + dest: /etc/bashrc + line: 'export PATH=/alma/{{ acs_tag }}/pyenv/shims:$PATH' diff --git a/deployment/__init__.py b/deployment/__init__.py index 7561a460..d28b02e0 100644 --- a/deployment/__init__.py +++ b/deployment/__init__.py @@ -521,7 +521,7 @@ def _docker_output(args, cwd=DEPLOYMENT_DIR, stdout=None, stderr=None, text=True def dockerImageExists(image): return _docker(["image", "inspect", image]) == 0 -def dockerBuild(image='discos-centos-7.9:latest'): +def dockerBuild(image='discos-centos-7.9:base'): """Build the docker image from ~/.deployment/Dockerfile using ~/.deployment as context.""" dockerfile_path = os.path.join(DEPLOYMENT_DIR, 'Dockerfile') if not os.path.exists(dockerfile_path): @@ -547,7 +547,7 @@ def dockerContainerRunning(name): def dockerEnsureContainer( name, ip, - image='discos-centos-7.9:latest', + image='discos-centos-7.9:base', network='discos_net' ): """Create container if missing, otherwise start it.""" @@ -686,3 +686,15 @@ def dockerLoadImage(imgfile): if not os.path.exists(imgfile): error(f"Image archive not found: {imgfile}") return _docker(['load', '-i', imgfile]) + +def dockerTagImage(source_image, target_image): + """Tag a local Docker image with another name.""" + return _docker(['tag', source_image, target_image]) + +def dockerPullImage(image): + """Pull a Docker image from a registry""" + return _docker(['pull', image]) + +def dockerPushImage(image): + """Push a Docker image to a registry.""" + return _docker(['push', image]) diff --git a/scripts/discos-container b/scripts/discos-container index 56ea68d4..3f701a08 100644 --- a/scripts/discos-container +++ b/scripts/discos-container @@ -23,8 +23,9 @@ import deployment CONTAINER = "manager" IMAGE = "discos-centos-7.9" +BASE_IMAGE = IMAGE + ":base" LATEST_IMAGE = IMAGE + ":latest" -PROVISIONED_IMAGE = IMAGE + ":provisioned" +GHCR_ORG = "ghcr.io/discos" NETWORK = "discos_net" SUBNET = "192.168.56.0/24" DEFAULT_INVENTORY = "development" @@ -57,9 +58,9 @@ def _ensure_docker_available(): def _resolve_image_from_tag(tag=None): if tag is None: - if deployment.dockerImageExists(PROVISIONED_IMAGE): - return PROVISIONED_IMAGE - return LATEST_IMAGE + if deployment.dockerImageExists(LATEST_IMAGE): + return LATEST_IMAGE + return BASE_IMAGE if ":" in tag: return tag @@ -67,7 +68,13 @@ def _resolve_image_from_tag(tag=None): return f"{IMAGE}:{tag}" -def build_image(image=LATEST_IMAGE): +def _resolve_ghcr_image_from_tag(tag="latest"): + local_image = _resolve_image_from_tag(tag) + image_name, image_tag = local_image.rsplit(":", 1) + return f"{GHCR_ORG}/{image_name}:{image_tag}" + + +def build_image(image=BASE_IMAGE): if deployment.dockerImageExists(image): print(f"Image {image} already exists.") return 0 @@ -80,7 +87,7 @@ def build_image(image=LATEST_IMAGE): ) -def remove_image(image=PROVISIONED_IMAGE): +def remove_image(image=LATEST_IMAGE): if not deployment.dockerImageExists(image): print(f"Image {image} not found.") return 1 @@ -103,6 +110,71 @@ def load_image(imgfile): ) +def push_image(tag="latest"): + local_image = _resolve_image_from_tag(tag) + remote_image = _resolve_ghcr_image_from_tag(tag) + + if not deployment.dockerImageExists(local_image): + print(f"Image {local_image} not found.") + return 1 + + rc = deployment.runWithProgress( + f"Tagging image {local_image} as {remote_image}", + deployment.dockerTagImage, + "error while tagging the requested image.", + local_image, + remote_image, + ) + if rc != 0: + return rc + + return deployment.runWithProgress( + f"Pushing image {remote_image}", + deployment.dockerPushImage, + "error while pushing the requested image.", + remote_image, + ) + + +def pull_image(tag="latest"): + local_image = _resolve_image_from_tag(tag) + remote_image = _resolve_ghcr_image_from_tag(tag) + + rc = deployment.runWithProgress( + f"Pulling image {remote_image}", + deployment.dockerPullImage, + "error while pulling the requested image.", + remote_image, + ) + if rc != 0: + return rc + + return deployment.runWithProgress( + f"Tagging image {remote_image} as {local_image}", + deployment.dockerTagImage, + "error while tagging the pulled image locally.", + remote_image, + local_image, + ) + + +def tag_image(source_tag, target_tag): + source_image = _resolve_image_from_tag(source_tag) + target_image = _resolve_image_from_tag(target_tag) + + if not deployment.dockerImageExists(source_image): + print(f"Image {source_image} not found.") + return 1 + + return deployment.runWithProgress( + f"Tagging image {source_image} as {target_image}", + deployment.dockerTagImage, + "error while tagging the requeste image.", + source_image, + target_image + ) + + def status_container(inventory=DEFAULT_INVENTORY): print("Container:") print("----------") @@ -120,16 +192,16 @@ def status_container(inventory=DEFAULT_INVENTORY): print("Images:") print("-------") + if deployment.dockerImageExists(BASE_IMAGE): + print(f"{BASE_IMAGE}: present") + else: + print(f"{BASE_IMAGE}: missing") + if deployment.dockerImageExists(LATEST_IMAGE): print(f"{LATEST_IMAGE}: present") else: print(f"{LATEST_IMAGE}: missing") - if deployment.dockerImageExists(PROVISIONED_IMAGE): - print(f"{PROVISIONED_IMAGE}: present") - else: - print(f"{PROVISIONED_IMAGE}: missing") - return 0 @@ -221,35 +293,17 @@ def restart_container(inventory=DEFAULT_INVENTORY): ) -def destroy_container(): +def destroy_container(force=False): if not deployment.dockerContainerExists(CONTAINER): print(f"Container {CONTAINER} not created.") - return 2 - - valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} - prompted = 0 - - while True: - if prompted == 3: - print("Aborted.") - return 0 + return 0 - prompted += 1 - sys.stdout.write( - f"Are you sure you want to destroy container {CONTAINER}? [y/N]: " - ) - sys.stdout.flush() + if not force: + print(f"Are you sure you want to destroy container {CONTAINER}? [y/N]: ", end="") choice = input().strip().lower() - - if choice == "": + if choice not in ("y", "yes"): print("Aborted.") - return 0 - - if choice in valid: - if not valid[choice]: - print("Aborted.") - return 0 - break + return 1 return deployment.runWithProgress( f"Destroying container {CONTAINER}", @@ -259,7 +313,7 @@ def destroy_container(): ) -def commit_container(tag="provisioned"): +def commit_container(tag="latest"): image = _resolve_image_from_tag(tag) if not deployment.dockerContainerExists(CONTAINER): @@ -275,7 +329,7 @@ def commit_container(tag="provisioned"): ) -def save_container(imgfile, tag="provisioned"): +def save_container(imgfile, tag="latest"): image = _resolve_image_from_tag(tag) if not deployment.dockerContainerExists(CONTAINER): @@ -294,6 +348,35 @@ def save_container(imgfile, tag="provisioned"): ) +def checkpoint_container(tag="latest", inventory=DEFAULT_INVENTORY, network=NETWORK, subnet=SUBNET): + image = _resolve_image_from_tag(tag) + + if not deployment.dockerContainerExists(CONTAINER): + print(f"Container {CONTAINER} not created.") + return 2 + + rc = deployment.runWithProgress( + f"Committing container {CONTAINER} into {image}", + deployment.dockerCommitContainer, + "error while committing the requested container.", + CONTAINER, + image, + ) + if rc != 0: + return rc + + rc = destroy_container(force=True) + if rc != 0: + return rc + + return create_container( + inventory=inventory, + tag=tag, + network=network, + subnet=subnet, + ) + + def _bind_command(parser, func, *arg_names): parser.set_defaults(func=func, arg_names=arg_names) @@ -325,16 +408,28 @@ def _parse_args(): help="image .tar filename", ) - def add_tag(p): + def add_tag(p, default=None): p.add_argument( "--tag", - default=None, + default=default, help=( "image tag to use. " - "If not provided: provisioned if available, latest otherwise" + "If not provided: latest if available, base otherwise" ), ) + def add_network(p): + p.add_argument( + "--network", + default=NETWORK, + help=f"Docker network to use (default: {NETWORK})", + ) + p.add_argument( + "--subnet", + default=SUBNET, + help=f"Docker subnet to use (default: {SUBNET})", + ) + p = subparsers.add_parser("build") _bind_command(p, build_image) @@ -371,22 +466,32 @@ def _parse_args(): p = subparsers.add_parser("create") add_inventory(p) - p.add_argument( - "--network", - default=NETWORK, - help=f"Docker network to use (default: {NETWORK})", - ) - p.add_argument( - "--subnet", - default=SUBNET, - help=f"Docker subnet to use (default: {SUBNET})", - ) + add_network(p) add_tag(p) _bind_command(p, create_container, "inventory", "tag", "network", "subnet") + p = subparsers.add_parser("checkpoint") + add_inventory(p) + add_network(p) + add_tag(p, default="latest") + _bind_command(p, checkpoint_container, "inventory", "tag", "network", "subnet") + p = subparsers.add_parser("destroy") _bind_command(p, destroy_container) + p = subparsers.add_parser("push") + add_tag(p, default="latest") + _bind_command(p, push_image, "tag") + + p = subparsers.add_parser("pull") + add_tag(p, default="latest") + _bind_command(p, pull_image, "tag") + + p = subparsers.add_parser("tag") + p.add_argument("source_tag") + p.add_argument("target_tag") + _bind_command(p, tag_image, "source_tag", "target_tag") + return parser.parse_args()