diff --git a/.github/utils/download_from_gdrive.py b/.github/utils/download_from_gdrive.py deleted file mode 100755 index 1f5c74f6..00000000 --- a/.github/utils/download_from_gdrive.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -from google.oauth2.credentials import Credentials -from google.auth.transport.requests import Request -from googleapiclient.discovery import build -from googleapiclient.http import MediaIoBaseDownload -import os -import io - -TOKEN_FILE = 'token.json' -CONTAINER_FILE_PATH = '/home/runner/discos_manager.tar' -SCOPES = ['https://www.googleapis.com/auth/drive.readonly'] - -# Create the token file from the GH Secret -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(os.environ.get('GOOGLE_DRIVE_TOKEN')) - -# Authenticate with the token and eventually update it -creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES) -if creds.expired and creds.refresh_token: - creds.refresh(Request()) - -service = build('drive', 'v3', credentials=creds) - -downloader = MediaIoBaseDownload( - io.FileIO(VM_FILE_PATH, 'wb'), - service.files().get_media( - fileId=os.environ.get('PROVISIONED_CONTAINER_GDRIVE_ID'), - ), - chunksize=64*1024*1024 -) -done = False -while not done: - status, done = downloader.next_chunk() - if status is not None: - print(f'Download progress: {int(status.progress() * 100)}%', flush=True) - -# Finally update the token file -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(creds.to_json()) diff --git a/.github/utils/gdrive_requirements.txt b/.github/utils/gdrive_requirements.txt deleted file mode 100644 index 7c686617..00000000 --- a/.github/utils/gdrive_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -google-auth -google-auth-oauthlib -google-auth-httplib2 -google-api-python-client diff --git a/.github/utils/upload_to_gdrive.py b/.github/utils/upload_to_gdrive.py deleted file mode 100755 index 67c818fe..00000000 --- a/.github/utils/upload_to_gdrive.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -from google.oauth2.credentials import Credentials -from google.auth.transport.requests import Request -from googleapiclient.discovery import build -from googleapiclient.http import MediaFileUpload -import os - -TOKEN_FILE = 'token.json' -CONTAINER_FILE_PATH = '/home/runner/discos_manager.tar' -SCOPES = [ - 'https://www.googleapis.com/auth/drive', - 'https://www.googleapis.com/auth/drive.file' -] - -# Create the token file from the GH Secret -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(os.environ.get('GOOGLE_DRIVE_TOKEN')) - -# Authenticate with the token and eventually update it -creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES) -if creds.expired and creds.refresh_token: - creds.refresh(Request()) - -# Prepare the files to be uploaded -service = build('drive', 'v3', credentials=creds) -container_media = MediaFileUpload( - CONTAINER_FILE_PATH, - mimetype='application/x-tar', - resumable=True, - chunksize=64*1024*1024 -) -request = service.files().update( - fileId=os.environ.get('PROVISIONED_CONTAINER_GDRIVE_ID'), - media_body=container_media, - fields='id' -) - -response = None -while response is None: - status, response = request.next_chunk() - if status is not None: - print(f'Upload progress: {int(status.progress() * 100)}%', flush=True) - -# Finally update the token file -with open(TOKEN_FILE, 'w') as tokenfile: - tokenfile.write(creds.to_json()) diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml index 46dc3994..1e7bc319 100644 --- a/.github/workflows/provision.yml +++ b/.github/workflows/provision.yml @@ -9,14 +9,15 @@ on: schedule: - cron: '0 6 * * 1' +permissions: + contents: read + packages: write + jobs: provision: + runs-on: ubuntu-latest env: REPOSITORY_TOKEN: "${{ secrets.DEPENDENCIES_TOKEN }}" - GH_TOKEN: "${{ secrets.GH_WORKFLOWS_TOKEN }}" - GOOGLE_DRIVE_TOKEN: "${{ secrets.GOOGLE_DRIVE_TOKEN }}" - PROVISIONED_CONTAINER_GDRIVE_ID: "${{ secrets.PROVISIONED_CONTAINER_GDRIVE_ID }}" - runs-on: ubuntu-latest steps: - name: Free up space uses: jlumbroso/free-disk-space@main @@ -44,16 +45,21 @@ jobs: else discos-deploy manager:development --docker --default-passwords fi - - name: Save provisioned image as tar artifact + - name: Login to GHCR + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Tag and push images to GHCR + if: github.event_name == 'push' && github.ref == 'refs/heads/master' run: | - discos-container save discos_manager.tar - working-directory: /home/runner + IMAGE_TAG="${GITHUB_REF_NAME}-${GITHUB_SHA::7}" + discos-container tag latest "${IMAGE_TAG}" + discos-container push --tag latest + discos-container push --tag "${IMAGE_TAG}" - name: Shutdown the container if: always() run: | discos-container stop || true - - name: Upload the Docker image to Google Drive - run: | - pip install -r .github/utils/gdrive_requirements.txt - python .github/utils/upload_to_gdrive.py - gh secret set GOOGLE_DRIVE_TOKEN --org discos --visibility selected --repos discos,deployment < token.json diff --git a/ansible/Medicina.yml b/ansible/Medicina.yml index f2244d40..7a2abef6 100644 --- a/ansible/Medicina.yml +++ b/ansible/Medicina.yml @@ -120,12 +120,22 @@ when: lustre_server_ip is defined -- name: Apply ACS configuration to ACS nodes +- name: Prepare for ACS setup tags: provision hosts: acs_meta remote_user: root roles: - - acs + - acs_prepare + - acs_environment + + +- name: Install ACS + tags: provision + hosts: acs_meta + remote_user: root + roles: + - acs_install + - discos_dependencies - name: Configure the manager machine diff --git a/ansible/Noto.yml b/ansible/Noto.yml index 85f4a175..766abe40 100644 --- a/ansible/Noto.yml +++ b/ansible/Noto.yml @@ -97,12 +97,22 @@ loop_var: username -- name: Apply ACS configuration to ACS nodes +- name: Prepare for ACS setup tags: provision hosts: manager remote_user: root roles: - - acs + - acs_prepare + - acs_environment + + +- name: Install ACS + tags: provision + hosts: manager + remote_user: root + roles: + - acs_install + - discos_dependencies - name: Configure the manager machine diff --git a/ansible/SRT.yml b/ansible/SRT.yml index ce339cf4..303a24fc 100644 --- a/ansible/SRT.yml +++ b/ansible/SRT.yml @@ -123,12 +123,22 @@ - sardara_remote_directory is defined -- name: Apply ACS configuration to ACS nodes +- name: Prepare for ACS setup tags: provision hosts: acs_meta remote_user: root roles: - - acs + - acs_prepare + - acs_environment + + +- name: Install ACS + tags: provision + hosts: acs_meta + remote_user: root + roles: + - acs_install + - discos_dependencies - name: Configure the manager machine diff --git a/ansible/development.yml b/ansible/development.yml index 9fa0ab03..49a04b98 100644 --- a/ansible/development.yml +++ b/ansible/development.yml @@ -111,12 +111,58 @@ loop_var: username -- name: Apply ACS configuration to ACS nodes +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" + + +- name: Prepare for ACS setup + tags: provision + hosts: manager + remote_user: root + roles: + - acs_prepare + - acs_environment + + +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" + + +- name: Install ACS tags: provision hosts: manager remote_user: root roles: - - acs + - acs_install + - discos_dependencies + + +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" - name: Configure the manager machine @@ -183,6 +229,18 @@ - vnc +- name: Commit current container checkpoint + tags: provision + hosts: manager + tasks: + - name: Commit current container checkpoint + ansible.builtin.command: "discos-container checkpoint" + delegate_to: localhost + run_once: true + changed_when: false + when: ansible_virtualization_type == "docker" + + - name: Deploy DISCOS tags: deploy hosts: manager diff --git a/ansible/roles/acs/tasks/java.yml b/ansible/roles/acs/tasks/java.yml deleted file mode 100644 index be28feaf..00000000 --- a/ansible/roles/acs/tasks/java.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - -- name: Install correct java yum packages - ansible.builtin.yum: - name: - - java-11-openjdk - - java-11-openjdk-headless - - java-11-openjdk-devel - state: present - - -- name: Fetch the installed yum packages - ansible.builtin.yum: - list: installed - register: package_facts - - -- name: Filter OpenJDK packages to be removed (non-Java 11) - ansible.builtin.set_fact: - openjdk_packages_to_remove: "{{ package_facts.results | - selectattr('name', 'match', '^java-[^-]+-openjdk(-(headless|devel))?$') | - selectattr('name', 'search', '^(?!java-11-openjdk)') | - map(attribute='name') | list }}" - - -- name: Uninstall OpenJDK packages different from Java 11 - ansible.builtin.yum: - name: "{{ item }}" - state: absent - with_items: "{{ openjdk_packages_to_remove }}" diff --git a/ansible/roles/acs/tasks/main.yml b/ansible/roles/acs/tasks/main.yml deleted file mode 100644 index faf894e1..00000000 --- a/ansible/roles/acs/tasks/main.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- - -- name: Install ACS yum packages - ansible.builtin.include_tasks: yum.yml - - -- name: Install correct JAVA packages - ansible.builtin.include_tasks: java.yml - - -- name: Configure the DISCOS environment - ansible.builtin.include_tasks: discos_environment.yml - - -- name: Install ACS - ansible.builtin.include_tasks: acs.yml - - -- name: Install the DISCOS dependencies - ansible.builtin.include_tasks: discos_dependencies.yml - - -- name: Install some utilities - ansible.builtin.include_tasks: utilities.yml - - -- name: Configure the monitoring for the receivers - ansible.builtin.include_tasks: receivers_monitor.yml - when: monitor_receivers diff --git a/ansible/roles/acs/tasks/receivers_monitor.yml b/ansible/roles/acs/tasks/receivers_monitor.yml deleted file mode 100644 index 480f4f08..00000000 --- a/ansible/roles/acs/tasks/receivers_monitor.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- - -- name: Create the /service/receivers directory - ansible.builtin.file: - path: "/service/receivers" - state: directory - owner: "{{ user.name }}" - group: "{{ user.group }}" - mode: "0755" - recurse: true - - -- name: Add receivers monitoring script to rc.local - ansible.builtin.blockinfile: - path: "/etc/rc.local" - state: present - marker: "######## Receivers monitoring {mark} ########" - block: | - if [ -f /{{ discos_sw_dir }}/introots/default/bin/receiversmonitor.py ]; then - runuser -l {{ user.name }} -c 'receiversmonitor.py &' - fi diff --git a/ansible/roles/acs/tasks/utilities.yml b/ansible/roles/acs/tasks/utilities.yml deleted file mode 100644 index cdf75058..00000000 --- a/ansible/roles/acs/tasks/utilities.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- - -- name: Install prospector (Python linter) - ansible.builtin.command: "pip install prospector" - become: true - become_user: "{{ user.name }}" - become_flags: "-i" - changed_when: true - - -- name: Install ipython - ansible.builtin.command: "pip install ipython" - become: true - become_user: "{{ user.name }}" - become_flags: "-i" - changed_when: true - - -- name: Create default ipython profile - ansible.builtin.command: "ipython profile create" - become: true - become_user: "{{ item.value.name }}" - become_flags: "-i" - with_dict: "{{ users }}" - changed_when: true - - -- name: Set the default application for html files - ansible.builtin.lineinfile: - path: "/usr/share/applications/mimeapps.list" - line: "{{ item }}" - state: present - create: true - mode: "0644" - with_items: - - "[Default Applications]" - - "text/html=firefox.desktop" - - -- name: Allow all users to have access to Python executables - ansible.builtin.lineinfile: - dest: /etc/bashrc - line: 'export PATH=/alma/{{ acs_tag }}/pyenv/shims:$PATH' diff --git a/ansible/roles/acs/tasks/discos_environment.yml b/ansible/roles/acs_environment/tasks/main.yml similarity index 95% rename from ansible/roles/acs/tasks/discos_environment.yml rename to ansible/roles/acs_environment/tasks/main.yml index 9cbfbfae..1c039db7 100644 --- a/ansible/roles/acs/tasks/discos_environment.yml +++ b/ansible/roles/acs_environment/tasks/main.yml @@ -158,11 +158,3 @@ fi no_log: true with_dict: "{{ users }}" - - -- name: Source scl_source - ansible.builtin.lineinfile: - dest: "{{ item }}" - line: 'source scl_source enable devtoolset-{{ devtoolset_release }}' - with_items: - - "/etc/bashrc" diff --git a/ansible/roles/acs/templates/_discos-check-branch b/ansible/roles/acs_environment/templates/_discos-check-branch similarity index 100% rename from ansible/roles/acs/templates/_discos-check-branch rename to ansible/roles/acs_environment/templates/_discos-check-branch diff --git a/ansible/roles/acs/templates/bashrc b/ansible/roles/acs_environment/templates/bashrc similarity index 98% rename from ansible/roles/acs/templates/bashrc rename to ansible/roles/acs_environment/templates/bashrc index 97f0f4f6..0da2586a 100644 --- a/ansible/roles/acs/templates/bashrc +++ b/ansible/roles/acs_environment/templates/bashrc @@ -54,7 +54,7 @@ make() { # Variables to export # =================== -export QTDIR={{ qt_dir }} +export QTDIR=/usr/lib64/qt4 export QTLIB=$QTDIR/lib export QTINC=/usr/include/Qt export qt_prefix=$QTLIB diff --git a/ansible/roles/acs/templates/dir_colors b/ansible/roles/acs_environment/templates/dir_colors similarity index 100% rename from ansible/roles/acs/templates/dir_colors rename to ansible/roles/acs_environment/templates/dir_colors diff --git a/ansible/roles/acs/templates/discos-get b/ansible/roles/acs_environment/templates/discos-get similarity index 100% rename from ansible/roles/acs/templates/discos-get rename to ansible/roles/acs_environment/templates/discos-get diff --git a/ansible/roles/acs/templates/discos-set b/ansible/roles/acs_environment/templates/discos-set similarity index 100% rename from ansible/roles/acs/templates/discos-set rename to ansible/roles/acs_environment/templates/discos-set diff --git a/ansible/roles/acs/templates/load_branch b/ansible/roles/acs_environment/templates/load_branch similarity index 100% rename from ansible/roles/acs/templates/load_branch rename to ansible/roles/acs_environment/templates/load_branch diff --git a/ansible/roles/acs/templates/prompt_command.sh b/ansible/roles/acs_environment/templates/prompt_command.sh similarity index 100% rename from ansible/roles/acs/templates/prompt_command.sh rename to ansible/roles/acs_environment/templates/prompt_command.sh diff --git a/ansible/roles/acs/templates/pyrc b/ansible/roles/acs_environment/templates/pyrc similarity index 100% rename from ansible/roles/acs/templates/pyrc rename to ansible/roles/acs_environment/templates/pyrc diff --git a/ansible/roles/acs/templates/vimrc b/ansible/roles/acs_environment/templates/vimrc similarity index 100% rename from ansible/roles/acs/templates/vimrc rename to ansible/roles/acs_environment/templates/vimrc diff --git a/ansible/roles/acs/defaults/main.yml b/ansible/roles/acs_install/defaults/main.yml similarity index 98% rename from ansible/roles/acs/defaults/main.yml rename to ansible/roles/acs_install/defaults/main.yml index ece253a4..015e4071 100644 --- a/ansible/roles/acs/defaults/main.yml +++ b/ansible/roles/acs_install/defaults/main.yml @@ -49,4 +49,3 @@ slalibrary_file: "SlaLibrary.tar.gz" ################ monitor_receivers: false -devtoolset_release: 8 diff --git a/ansible/roles/acs/tasks/acs.yml b/ansible/roles/acs_install/tasks/main.yml similarity index 93% rename from ansible/roles/acs/tasks/acs.yml rename to ansible/roles/acs_install/tasks/main.yml index c631d4db..c072bc19 100644 --- a/ansible/roles/acs/tasks/acs.yml +++ b/ansible/roles/acs_install/tasks/main.yml @@ -1,25 +1,5 @@ --- -- name: Check in which path Java is installed - ansible.builtin.shell: "readlink -f $(which java)" - register: java_home - changed_when: false - - -- name: Make sure /usr/java directory exists - ansible.builtin.file: - path: "/usr/java" - state: directory - mode: "0755" - - -- name: Create a symlink to the correct Java home path, needed by ACS - ansible.builtin.file: - src: "/usr/lib/jvm/java-openjdk" - dest: "/usr/java/default" - state: link - - - name: Create the /alma directory ansible.builtin.file: path: "/alma" diff --git a/ansible/roles/acs/tasks/patches/ACS-2021DEC.yml b/ansible/roles/acs_install/tasks/patches/ACS-2021DEC.yml similarity index 100% rename from ansible/roles/acs/tasks/patches/ACS-2021DEC.yml rename to ansible/roles/acs_install/tasks/patches/ACS-2021DEC.yml diff --git a/ansible/roles/acs/templates/acsStartContainerGdb b/ansible/roles/acs_install/templates/acsStartContainerGdb similarity index 100% rename from ansible/roles/acs/templates/acsStartContainerGdb rename to ansible/roles/acs_install/templates/acsStartContainerGdb diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/BaciHelper.py b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/BaciHelper.py similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/BaciHelper.py rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/BaciHelper.py diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/ExecuteContainer.java b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/ExecuteContainer.java similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/ExecuteContainer.java rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/ExecuteContainer.java diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/Executor.java b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/Executor.java similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/Executor.java rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/Executor.java diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/WDALImpl.java b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/WDALImpl.java similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/WDALImpl.java rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/WDALImpl.java diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildDDS b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildDDS similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildDDS rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildDDS diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildOmniORB b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildOmniORB similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildOmniORB rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildOmniORB diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildPyModules b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPyModules similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildPyModules rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPyModules diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/buildPython b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPython similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/buildPython rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/buildPython diff --git a/ansible/roles/acs/templates/patches/ACS-2021DEC/enumpropAlarm.i b/ansible/roles/acs_install/templates/patches/ACS-2021DEC/enumpropAlarm.i similarity index 100% rename from ansible/roles/acs/templates/patches/ACS-2021DEC/enumpropAlarm.i rename to ansible/roles/acs_install/templates/patches/ACS-2021DEC/enumpropAlarm.i diff --git a/ansible/roles/acs_prepare/defaults/main.yml b/ansible/roles/acs_prepare/defaults/main.yml new file mode 100644 index 00000000..352cd177 --- /dev/null +++ b/ansible/roles/acs_prepare/defaults/main.yml @@ -0,0 +1,3 @@ +--- + +devtoolset_release: 8 diff --git a/ansible/roles/acs/tasks/yum.yml b/ansible/roles/acs_prepare/tasks/main.yml similarity index 70% rename from ansible/roles/acs/tasks/yum.yml rename to ansible/roles/acs_prepare/tasks/main.yml index 4e439bf7..5519106d 100644 --- a/ansible/roles/acs/tasks/yum.yml +++ b/ansible/roles/acs_prepare/tasks/main.yml @@ -115,10 +115,18 @@ - name: Disable the devtoolset-{{ devtoolset_release }} sudo command ansible.builtin.file: - path: "/opt/rh/devtoolset-8/root/usr/bin/sudo" + path: "/opt/rh/devtoolset-{{ devtoolset_release }}/root/usr/bin/sudo" mode: "0444" +- name: Source scl_source + ansible.builtin.lineinfile: + dest: "{{ item }}" + line: 'source scl_source enable devtoolset-{{ devtoolset_release }}' + with_items: + - "/etc/bashrc" + + - name: Make sure the default git yum package is not installed ansible.builtin.yum: name: @@ -171,3 +179,53 @@ src: /usr/bin/cmake3 dest: /usr/bin/cmake state: link + + +- name: Install correct java yum packages + ansible.builtin.yum: + name: + - java-11-openjdk + - java-11-openjdk-headless + - java-11-openjdk-devel + state: present + + +- name: Fetch the installed yum packages + ansible.builtin.yum: + list: installed + register: package_facts + + +- name: Filter OpenJDK packages to be removed (non-Java 11) + ansible.builtin.set_fact: + openjdk_packages_to_remove: "{{ package_facts.results | + selectattr('name', 'match', '^java-[^-]+-openjdk(-(headless|devel))?$') | + selectattr('name', 'search', '^(?!java-11-openjdk)') | + map(attribute='name') | list }}" + + +- name: Uninstall OpenJDK packages different from Java 11 + ansible.builtin.yum: + name: "{{ item }}" + state: absent + with_items: "{{ openjdk_packages_to_remove }}" + + +- name: Check in which path Java is installed + ansible.builtin.shell: "readlink -f $(which java)" + register: java_home + changed_when: false + + +- name: Make sure /usr/java directory exists + ansible.builtin.file: + path: "/usr/java" + state: directory + mode: "0755" + + +- name: Create a symlink to the correct Java home path, needed by ACS + ansible.builtin.file: + src: "/usr/lib/jvm/java-openjdk" + dest: "/usr/java/default" + state: link diff --git a/ansible/roles/discos_dependencies/defaults/main.yml b/ansible/roles/discos_dependencies/defaults/main.yml new file mode 100644 index 00000000..fc9e3238 --- /dev/null +++ b/ansible/roles/discos_dependencies/defaults/main.yml @@ -0,0 +1,24 @@ +--- + +cfitsio: { file: 'cfitsio3370.tar.gz', build_dir: 'cfitsio' } +ccfits: { file: 'CCfits-2.4.tar.gz', build_dir: 'CCfits' } +fv: { file: 'fv5.4_pc_linux64.tar.gz' } +modbus: { file: 'libmodbus-3.0.6.tar.gz', build_dir: 'libmodbus-3.0.6' } +f2c: { file: 'f2c.zip'} + +googletest_repo: "447d58b4ee8ea96b4757a5bb5f0b3be75af6c2a1" +googletest_build_dir: "googletest-{{ googletest_repo }}" +googletest_file: "{{ googletest_build_dir }}.zip" +gmock_build: "{{ remote_build_path }}/{{ googletest_build_dir }}/googlemock" +gtest_build: "{{ remote_build_path }}/{{ googletest_build_dir }}/googletest" + +sip_build_dir: sip-4.19.2 +sip_file: "{{ sip_build_dir }}.tar.gz" + +pyqt_build_dir: PyQt4_gpl_x11-4.12 +pyqt_file: "{{ pyqt_build_dir }}.tar.gz" + +pyqtqwt_build_dir: PyQt-Qwt +pyqtqwt_file: "{{ pyqtqwt_build_dir }}.zip" + +slalibrary_file: "SlaLibrary.tar.gz" diff --git a/ansible/roles/acs/tasks/discos_dependencies.yml b/ansible/roles/discos_dependencies/tasks/main.yml similarity index 95% rename from ansible/roles/acs/tasks/discos_dependencies.yml rename to ansible/roles/discos_dependencies/tasks/main.yml index 2c17f90a..38e874a1 100644 --- a/ansible/roles/acs/tasks/discos_dependencies.yml +++ b/ansible/roles/discos_dependencies/tasks/main.yml @@ -960,3 +960,51 @@ become_user: "{{ user.name }}" become_flags: "-i" changed_when: true + + +########### +# Utilities +########### + + +- name: Install prospector (Python linter) + ansible.builtin.command: "pip install prospector" + become: true + become_user: "{{ user.name }}" + become_flags: "-i" + changed_when: true + + +- name: Install ipython + ansible.builtin.command: "pip install ipython" + become: true + become_user: "{{ user.name }}" + become_flags: "-i" + changed_when: true + + +- name: Create default ipython profile + ansible.builtin.command: "ipython profile create" + become: true + become_user: "{{ item.value.name }}" + become_flags: "-i" + with_dict: "{{ users }}" + changed_when: true + + +- name: Set the default application for html files + ansible.builtin.lineinfile: + path: "/usr/share/applications/mimeapps.list" + line: "{{ item }}" + state: present + create: true + mode: "0644" + with_items: + - "[Default Applications]" + - "text/html=firefox.desktop" + + +- name: Allow all users to have access to Python executables + ansible.builtin.lineinfile: + dest: /etc/bashrc + line: 'export PATH=/alma/{{ acs_tag }}/pyenv/shims:$PATH' diff --git a/deployment/__init__.py b/deployment/__init__.py index 7561a460..d28b02e0 100644 --- a/deployment/__init__.py +++ b/deployment/__init__.py @@ -521,7 +521,7 @@ def _docker_output(args, cwd=DEPLOYMENT_DIR, stdout=None, stderr=None, text=True def dockerImageExists(image): return _docker(["image", "inspect", image]) == 0 -def dockerBuild(image='discos-centos-7.9:latest'): +def dockerBuild(image='discos-centos-7.9:base'): """Build the docker image from ~/.deployment/Dockerfile using ~/.deployment as context.""" dockerfile_path = os.path.join(DEPLOYMENT_DIR, 'Dockerfile') if not os.path.exists(dockerfile_path): @@ -547,7 +547,7 @@ def dockerContainerRunning(name): def dockerEnsureContainer( name, ip, - image='discos-centos-7.9:latest', + image='discos-centos-7.9:base', network='discos_net' ): """Create container if missing, otherwise start it.""" @@ -686,3 +686,15 @@ def dockerLoadImage(imgfile): if not os.path.exists(imgfile): error(f"Image archive not found: {imgfile}") return _docker(['load', '-i', imgfile]) + +def dockerTagImage(source_image, target_image): + """Tag a local Docker image with another name.""" + return _docker(['tag', source_image, target_image]) + +def dockerPullImage(image): + """Pull a Docker image from a registry""" + return _docker(['pull', image]) + +def dockerPushImage(image): + """Push a Docker image to a registry.""" + return _docker(['push', image]) diff --git a/scripts/discos-container b/scripts/discos-container index 56ea68d4..3f701a08 100644 --- a/scripts/discos-container +++ b/scripts/discos-container @@ -23,8 +23,9 @@ import deployment CONTAINER = "manager" IMAGE = "discos-centos-7.9" +BASE_IMAGE = IMAGE + ":base" LATEST_IMAGE = IMAGE + ":latest" -PROVISIONED_IMAGE = IMAGE + ":provisioned" +GHCR_ORG = "ghcr.io/discos" NETWORK = "discos_net" SUBNET = "192.168.56.0/24" DEFAULT_INVENTORY = "development" @@ -57,9 +58,9 @@ def _ensure_docker_available(): def _resolve_image_from_tag(tag=None): if tag is None: - if deployment.dockerImageExists(PROVISIONED_IMAGE): - return PROVISIONED_IMAGE - return LATEST_IMAGE + if deployment.dockerImageExists(LATEST_IMAGE): + return LATEST_IMAGE + return BASE_IMAGE if ":" in tag: return tag @@ -67,7 +68,13 @@ def _resolve_image_from_tag(tag=None): return f"{IMAGE}:{tag}" -def build_image(image=LATEST_IMAGE): +def _resolve_ghcr_image_from_tag(tag="latest"): + local_image = _resolve_image_from_tag(tag) + image_name, image_tag = local_image.rsplit(":", 1) + return f"{GHCR_ORG}/{image_name}:{image_tag}" + + +def build_image(image=BASE_IMAGE): if deployment.dockerImageExists(image): print(f"Image {image} already exists.") return 0 @@ -80,7 +87,7 @@ def build_image(image=LATEST_IMAGE): ) -def remove_image(image=PROVISIONED_IMAGE): +def remove_image(image=LATEST_IMAGE): if not deployment.dockerImageExists(image): print(f"Image {image} not found.") return 1 @@ -103,6 +110,71 @@ def load_image(imgfile): ) +def push_image(tag="latest"): + local_image = _resolve_image_from_tag(tag) + remote_image = _resolve_ghcr_image_from_tag(tag) + + if not deployment.dockerImageExists(local_image): + print(f"Image {local_image} not found.") + return 1 + + rc = deployment.runWithProgress( + f"Tagging image {local_image} as {remote_image}", + deployment.dockerTagImage, + "error while tagging the requested image.", + local_image, + remote_image, + ) + if rc != 0: + return rc + + return deployment.runWithProgress( + f"Pushing image {remote_image}", + deployment.dockerPushImage, + "error while pushing the requested image.", + remote_image, + ) + + +def pull_image(tag="latest"): + local_image = _resolve_image_from_tag(tag) + remote_image = _resolve_ghcr_image_from_tag(tag) + + rc = deployment.runWithProgress( + f"Pulling image {remote_image}", + deployment.dockerPullImage, + "error while pulling the requested image.", + remote_image, + ) + if rc != 0: + return rc + + return deployment.runWithProgress( + f"Tagging image {remote_image} as {local_image}", + deployment.dockerTagImage, + "error while tagging the pulled image locally.", + remote_image, + local_image, + ) + + +def tag_image(source_tag, target_tag): + source_image = _resolve_image_from_tag(source_tag) + target_image = _resolve_image_from_tag(target_tag) + + if not deployment.dockerImageExists(source_image): + print(f"Image {source_image} not found.") + return 1 + + return deployment.runWithProgress( + f"Tagging image {source_image} as {target_image}", + deployment.dockerTagImage, + "error while tagging the requeste image.", + source_image, + target_image + ) + + def status_container(inventory=DEFAULT_INVENTORY): print("Container:") print("----------") @@ -120,16 +192,16 @@ def status_container(inventory=DEFAULT_INVENTORY): print("Images:") print("-------") + if deployment.dockerImageExists(BASE_IMAGE): + print(f"{BASE_IMAGE}: present") + else: + print(f"{BASE_IMAGE}: missing") + if deployment.dockerImageExists(LATEST_IMAGE): print(f"{LATEST_IMAGE}: present") else: print(f"{LATEST_IMAGE}: missing") - if deployment.dockerImageExists(PROVISIONED_IMAGE): - print(f"{PROVISIONED_IMAGE}: present") - else: - print(f"{PROVISIONED_IMAGE}: missing") - return 0 @@ -221,35 +293,17 @@ def restart_container(inventory=DEFAULT_INVENTORY): ) -def destroy_container(): +def destroy_container(force=False): if not deployment.dockerContainerExists(CONTAINER): print(f"Container {CONTAINER} not created.") - return 2 - - valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} - prompted = 0 - - while True: - if prompted == 3: - print("Aborted.") - return 0 + return 0 - prompted += 1 - sys.stdout.write( - f"Are you sure you want to destroy container {CONTAINER}? [y/N]: " - ) - sys.stdout.flush() + if not force: + print(f"Are you sure you want to destroy container {CONTAINER}? [y/N]: ", end="") choice = input().strip().lower() - - if choice == "": + if choice not in ("y", "yes"): print("Aborted.") - return 0 - - if choice in valid: - if not valid[choice]: - print("Aborted.") - return 0 - break + return 1 return deployment.runWithProgress( f"Destroying container {CONTAINER}", @@ -259,7 +313,7 @@ def destroy_container(): ) -def commit_container(tag="provisioned"): +def commit_container(tag="latest"): image = _resolve_image_from_tag(tag) if not deployment.dockerContainerExists(CONTAINER): @@ -275,7 +329,7 @@ def commit_container(tag="provisioned"): ) -def save_container(imgfile, tag="provisioned"): +def save_container(imgfile, tag="latest"): image = _resolve_image_from_tag(tag) if not deployment.dockerContainerExists(CONTAINER): @@ -294,6 +348,35 @@ def save_container(imgfile, tag="provisioned"): ) +def checkpoint_container(tag="latest", inventory=DEFAULT_INVENTORY, network=NETWORK, subnet=SUBNET): + image = _resolve_image_from_tag(tag) + + if not deployment.dockerContainerExists(CONTAINER): + print(f"Container {CONTAINER} not created.") + return 2 + + rc = deployment.runWithProgress( + f"Committing container {CONTAINER} into {image}", + deployment.dockerCommitContainer, + "error while committing the requested container.", + CONTAINER, + image, + ) + if rc != 0: + return rc + + rc = destroy_container(force=True) + if rc != 0: + return rc + + return create_container( + inventory=inventory, + tag=tag, + network=network, + subnet=subnet, + ) + + def _bind_command(parser, func, *arg_names): parser.set_defaults(func=func, arg_names=arg_names) @@ -325,16 +408,28 @@ def _parse_args(): help="image .tar filename", ) - def add_tag(p): + def add_tag(p, default=None): p.add_argument( "--tag", - default=None, + default=default, help=( "image tag to use. " - "If not provided: provisioned if available, latest otherwise" + "If not provided: latest if available, base otherwise" ), ) + def add_network(p): + p.add_argument( + "--network", + default=NETWORK, + help=f"Docker network to use (default: {NETWORK})", + ) + p.add_argument( + "--subnet", + default=SUBNET, + help=f"Docker subnet to use (default: {SUBNET})", + ) + p = subparsers.add_parser("build") _bind_command(p, build_image) @@ -371,22 +466,32 @@ def _parse_args(): p = subparsers.add_parser("create") add_inventory(p) - p.add_argument( - "--network", - default=NETWORK, - help=f"Docker network to use (default: {NETWORK})", - ) - p.add_argument( - "--subnet", - default=SUBNET, - help=f"Docker subnet to use (default: {SUBNET})", - ) + add_network(p) add_tag(p) _bind_command(p, create_container, "inventory", "tag", "network", "subnet") + p = subparsers.add_parser("checkpoint") + add_inventory(p) + add_network(p) + add_tag(p, default="latest") + _bind_command(p, checkpoint_container, "inventory", "tag", "network", "subnet") + p = subparsers.add_parser("destroy") _bind_command(p, destroy_container) + p = subparsers.add_parser("push") + add_tag(p, default="latest") + _bind_command(p, push_image, "tag") + + p = subparsers.add_parser("pull") + add_tag(p, default="latest") + _bind_command(p, pull_image, "tag") + + p = subparsers.add_parser("tag") + p.add_argument("source_tag") + p.add_argument("target_tag") + _bind_command(p, tag_image, "source_tag", "target_tag") + return parser.parse_args()