Darwin Setup

This page documents the main steps used to set up Darwin.

Head Node OS Installation

  • Download Rocky Linux 9.8 x86_64 DVD ISO
  • Install ISO
  • Add password for root, unlock root, enable SSH with password
  • Add 'localadmin' administrator. DO NOT USE '!' OR '$' IN PASSWORD
  • Disable IPv6 on all network devices
  • Set hostname to 'darwin'
  • Merge '/' and '/home' (remove '/home' partition)

Head Node Setup

  • Connect to the head node 
    ssh localadmin@192.168.1.152

sudo -i
  • Enable persistent logging 
    systemd-tmpfiles --create --prefix /var/log/journal

mkdir -p /etc/systemd/journald.conf.d

printf '%s\n' '[Journal]' 'Storage=persistent' \
   >/etc/systemd/journald.conf.d/persistent.conf

systemctl restart systemd-journald
  • Set up kdump and netconsole 
    install \
   -d \
   -m 0755 \
   /var/log/netconsole \
   /var/crash/compute

tar -xzpf /root/assets/netconsole.tar.gz -C /

systemctl daemon-reload

cat > /etc/exports.d/kdump-compute.exports << 'EOF'
/var/crash/compute 10.141.0.0/16(rw,sync,no_subtree_check,no_root_squash)
EOF

exportfs -ra

systemctl reload nfs-server

systemctl restart rsyslog
  • Install the required packages 
    dnf install -y \
   epel-release \
   https://yum.osc.edu/ondemand/3.1/ondemand-release-compute-3.1-1.el9.noarch.rpm \
   https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm &&

dnf config-manager --set-enabled crb &&

dnf --enablerepo=elrepo-kernel install -y kmod-mlx4 &&

dnf install -y \
   git-core \
   gh \
   opensm \
   ripgrep \
   tmux \
   fio \
   iotop \
   selinux-policy-devel \
   fail2ban fail2ban-firewalld \
   turbovnc &&

systemctl enable --now opensm fail2ban &&

reboot
  • Disable GUI autoupdates 
    cat > /etc/dconf/db/local.d/00-gnome-software << EOF
[org/gnome/software]
allow-updates=false
download-updates=false
download-updates-notify=false
EOF

cat > /etc/dconf/db/local.d/locks/00-gnome-software << EOF
/org/gnome/software/allow-updates
/org/gnome/software/download-updates
/org/gnome/software/download-updates-notify
EOF

dconf update
  • Add network connections 
    nmcli con add \
   type ethernet \
   ifname eno2 \
   con-name internal \
   ipv4.method manual \
   ipv4.addresses 10.141.255.254/16 \
   ipv4.gateway "" \
   ipv6.method disable

nmcli con up internal

nmcli con add \
   type ethernet \
   ifname enp2s0 \
   con-name bmc \
   ipv4.method manual \
   ipv4.addresses 10.148.255.254/16 \
   ipv4.gateway "" \
   ipv6.method disable

nmcli con up bmc

nmcli con add \
   type infiniband \
   ifname ibp3s0 \
   con-name ib \
   ipv4.method manual \
   ipv4.addresses 10.149.255.254/16 \
   ipv4.gateway "" \
   ipv6.method disable

nmcli con up ib
  • Set up RAID 
    cat >> /etc/fstab/ << EOF
UUID=b5788192-0303-4ef5-944e-ac6354bd0938 /trinity/home ext4 defaults 0 0
UUID=fab54cdf-d1c6-4d7c-ae83-649a480ab3cb /trinity/structbio_home ext4 defaults,noatime 0 0
UUID=d27804ca-99df-4b46-ae65-1f04bfcb80fa /opt ext4 defaults 0 2
EOF

systemctl daemon-reload
cat > /etc/mdadm.conf << EOF
ARRAY /dev/md/newton:2 metadata=1.2 UUID=080b6df8:ddb3019f:665d341e:5fcaabc7
ARRAY /dev/md/struct_biotech metadata=1.2 spares=1 UUID=72c25cfc:fc70a47f:3d22ce14:7e5a00cc
EOF
  • If a proxy is required
    • Set environment variables 
      cat >> ~/.bashrc << 'EOF'
proxy() {
 local PROXY="http://localhost:10808"
 export http_proxy="$PROXY"
 export https_proxy="$PROXY"
 export HTTP_PROXY="$PROXY"
 export HTTPS_PROXY="$PROXY"
}
EOF

. ~/.bashrc
    • Forward a proxy port (use any free port if a VPN is enabled) 
      ssh -N -R 17890:127.0.0.1:<PORT> root@192.168.1.152

      OR 

      cat > /etc/profile.d/aliases.sh << 'EOF'
case "$-" in
 *i*) ;;
 *) return 0 2>/dev/null || exit 0 ;;
esac

alias vpn="ip route replace default via 192.168.1.182 dev eno1"
alias disvpn="ip route del default via 192.168.1.182 dev eno1"
EOF

chmod 700 /etc/profile.d/aliases.sh
  • Download trinityX 
    cd

git clone http://github.com/clustervision/trinityX &&

cd trinityX
  • Install trinityX 
    cd ~/trinityX

bash prepare.sh &&

cd site

Answer 'y' to a message about SELinux.

Reboot and repeat script run if asked to.

Answer 'n' to a message about ZFS.

  • Disable SSL 
    sed -i \
   -e 's/enable_ssl: true/enable_ssl: false/g' \
   controller.yml

sed -i \
   '/grafana_prometheus_url/s/:9090"/:9090\/prometheus"/' \
   /root/trinityX/site/controller.yml
  • Enable RDMA for NFS and mount /home, /shared, /opt, /ohpc 
    sed -i \
   "/^trix_ctrl_hostname:/a \

# -----------------------------------------------------------------------\
# Storage node IP address.\
# Used to mount shared filesystems (home, opt, shared, scratch) via NFS RDMA.\
# In a single-node setup, this should match the controller's InfiniBand IP.\
\
trix_storage_ip: 10.149.255.254" \
   group_vars/all.yml.example &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/# easybuild/,/disabled:/ {
       /disabled:/a \
       # opt ------------------------------------------------------------------------------ \
       - path: '\''/opt'\'' \
         options: '\''*(rw,async,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\'' \
       # structbio_home ------------------------------------------------------------------------------ \
       - path: '\''/trinity/structbio_home'\'' \
         options: '\''*(rw,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\'' \
    }' \
   controller.yml &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/remote:/s/{{ trix_ctrl_hostname }}/{{ trix_storage_ip }}/g' \
   -e "/- path: '{{ trix_shared }}'/,/options: 'defaults,nfsvers=4.2,ro,retrans=4,_netdev'/d" \
   -e "/- path: '{{ trix_home }}'/,/options: 'defaults,nfsvers=4.2,rw,retrans=4,noatime,_netdev'/d" \
   -e '1,/nfs_mounts:/!b' \
   -e '/nfs_mounts:/a \
   - path: '\''{{ trix_shared }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_shared }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,ro,retrans=4,noatime,_netdev'\''\
   - path: '\''{{ trix_home }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_home }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/trinity/structbio_home'\''\
     remote: '\''{{ trix_storage_ip }}:/trinity/structbio_home'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/opt'\''\
     remote: '\''{{ trix_storage_ip }}:/opt'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\'''\
   imports/trinity-redhat-image-setup.yml
  • Enable PAM Slurm adopt module 
    sed -i \
   -e '/^account[[:space:]]\{1,\}include[[:space:]]\{1,\}password-auth$/a\account    required     pam_slurm_adopt.so' \
   /root/trinityX/site/roles/trinity/slurm/files/pam-sshd

sed -i \
   's/^enable_slurm_pam:[[:space:]]*false$/enable_slurm_pam: true/' \
   /root/trinityX/site/roles/trinity/slurm/defaults/main.yml
  • Fix Prometheus path 
    sed -i \
   $'/when: luna2_daemon_pip is undefined/a\

- name: Patch luna2 daemon Prometheus route prefix\
     ansible.builtin.replace:\
   path: '\''{{ luna2_python_dir.stdout }}/site-packages/daemon/plugins/import/prometheus_hw_rules.py'\''\
   regexp: '\''self\\.prometheus_url = "https://localhost:9090"'\''\
   replace: '\''self.prometheus_url = "http://localhost:9090/prometheus"'\''' \
   /root/trinityX/site/roles/trinity/luna2/tasks/controller.yml
  • Prepare Ansible inventory 
    cat > hosts << EOF
[controllers]
darwin ansible_connection=local
EOF
  • Prepare the head node playbook 
    cp group_vars/all.yml.example group_vars/all.yml

sed -i \
   -e 's/^trix_ctrl1_hostname:.*/trix_ctrl1_hostname: darwin/' \
   -e "s|^trix_external_fqdn:.*|trix_external_fqdn: 'portal.darwin.bioeng.ru'|" \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.8\.8"/\1- "192.168.1.3"/' \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.4\.4"/\1- "192.168.1.1"/' \
   -e 's/^luna_password:.*/luna_password: .../' \
   -e 's/^luna_protocol:.*/luna_protocol: http/' \
   -e 's/^firewalld_public_interfaces:.*/firewalld_public_interfaces: [eno1]/' \
   -e 's/^firewalld_trusted_interfaces:.*/firewalld_trusted_interfaces: [eno2, enp23s0, ibp180s0]/' \
   -e 's/^firewalld_public_tcp_ports:.*/firewalld_public_tcp_ports: [22, 80, 443]/' \
   group_vars/all.yml
  • Patch playbook to install OOD apps idempotently 
    sed -i -f - /root/trinityX/site/roles/trinity/ood-apps/tasks/app.yml <<'SED'
/^    - name: Make folder for app /,/^      notify: Run update_ood$/ {
 /^      notify: Run update_ood$/ a\

   - name: Ensure web server can traverse trinity app directory\
     ansible.posix.acl:\
       path: "{{ app_path }}"\
       entity: apache\
       etype: user\
       permissions: rx\
       state: present\
     when:\
       - app_enabled\
       - app.get('type') == 'trinity'\
     notify: Run update_ood
}
SED
  • Launch the head node playbook 
    ansible-playbook controller.yml
  • Enable RDMA for NFS and mount /home, /shared, /opt, /ohpc 
    sed -i \
   "/^trix_ctrl_hostname:/a \\

# -----------------------------------------------------------------------\
# Storage node IP address.\
# Used to mount shared filesystems (home, opt, shared, scratch) via NFS RDMA.\
# In a single-node setup, this should match the controller's InfiniBand IP.\
\
trix_storage_ip: 10.149.255.254" \
   group_vars/all.yml.example &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/# easybuild/,/disabled:/ {
       /disabled:/a \
       # opt ------------------------------------------------------------------------------ \
       - path: '\''/opt'\'' \
         options: '\''*(rw,async,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\'' \
       # structbio_home ------------------------------------------------------------------------------ \
       - path: '\''/trinity/structbio_home'\'' \
         options: '\''*(rw,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\''
    }' \
   controller.yml &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/remote:/s/{{ trix_ctrl_hostname }}/{{ trix_storage_ip }}/g' \
   -e "/- path: '{{ trix_shared }}'/,/options: 'defaults,nfsvers=4.2,ro,retrans=4,_netdev'/d" \
   -e "/- path: '{{ trix_home }}'/,/options: 'defaults,nfsvers=4.2,rw,retrans=4,noatime,_netdev'/d" \
   -e '1,/nfs_mounts:/!b' \
   -e '/nfs_mounts:/a \
   - path: '\''{{ trix_shared }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_shared }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,ro,retrans=4,noatime,_netdev'\''\
   - path: '\''{{ trix_home }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_home }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/trinity/structbio_home'\''\
     remote: '\''{{ trix_storage_ip }}:/trinity/structbio_home'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/opt'\''\
     remote: '\''{{ trix_storage_ip }}:/opt'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\'''\
   imports/trinity-redhat-image-setup.yml
  • Enable PAM Slurm adopt module 
    sed -i \
   -e '/^account[[:space:]]\{1,\}include[[:space:]]\{1,\}password-auth$/a\account    required     pam_slurm_adopt.so' \
   /root/trinityX/site/roles/trinity/slurm/files/pam-sshd

sed -i \
   's/^enable_slurm_pam:[[:space:]]*false$/enable_slurm_pam: true/' \
   /root/trinityX/site/roles/trinity/slurm/defaults/main.yml
  • Fix Prometheus path 
    sed -i \
   $'/when: luna2_daemon_pip is undefined/a\

- name: Patch luna2 daemon Prometheus route prefix\
     ansible.builtin.replace:\
   path: '\''{{ luna2_python_dir.stdout }}/site-packages/daemon/plugins/import/prometheus_hw_rules.py'\''\
   regexp: '\''self\\.prometheus_url = "https://localhost:9090"'\''\
   replace: '\''self.prometheus_url = "http://localhost:9090/prometheus"'\''' \
   /root/trinityX/site/roles/trinity/luna2/tasks/controller.yml
  • Prepare Ansible inventory 
    cat > hosts << EOF
[controllers]
darwin ansible_connection=local
EOF
  • Prepare the head node playbook 
    cp group_vars/all.yml.example group_vars/all.yml

sed -i \
   -e 's/^trix_ctrl1_hostname:.*/trix_ctrl1_hostname: darwin/' \
   -e "s|^trix_external_fqdn:.*|trix_external_fqdn: '\''portal.darwin.bioeng.ru'\''|" \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.8\.8"/\1- "192.168.1.3"/' \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.4\.4"/\1- "192.168.1.1"/' \
   -e 's/^luna_password:.*/luna_password: hKpT5APf@jfp5g/' \
   -e 's/^luna_protocol:.*/luna_protocol: http/' \
   -e 's/^firewalld_public_interfaces:.*/firewalld_public_interfaces: [eno1]/' \
   -e 's/^firewalld_trusted_interfaces:.*/firewalld_trusted_interfaces: [eno2, enp23s0, ibp180s0]/' \
   -e 's/^firewalld_public_tcp_ports:.*/firewalld_public_tcp_ports: [22, 80, 443]/' \
   group_vars/all.yml
  • Patch playbook to install OOD apps idempotently 
    sed -i -f - /root/trinityX/site/roles/trinity/ood-apps/tasks/app.yml <<'SED'
/^    - name: Make folder for app /,/^      notify: Run update_ood$/ {
 /^      notify: Run update_ood$/ a\

   - name: Ensure web server can traverse trinity app directory\
     ansible.posix.acl:\
       path: "{{ app_path }}"\
       entity: apache\
       etype: user\
       permissions: rx\
       state: present\
     when:\
       - app_enabled\
       - app.get('type') == 'trinity'\
     notify: Run update_ood
}
SED
  • Launch the head node playbook 
    ansible-playbook controller.yml
  • Limit non-admin users on the head node by 0.5 CPU and 256 MB RAM 
    cat > /usr/local/bin/select-slice.sh << EOF
#!/bin/bash

USER_NAME="${PAM_USER:-}"
[ -z "$USER_NAME" ] && exit 0

UID_NUM="$(id -u "$USER_NAME" 2>/dev/null)" || exit 0

export DBUS_SYSTEM_BUS_ADDRESS="unix:path=/run/dbus/system_bus_socket"
UNIT="user-${UID_NUM}.slice"

if id -nG "$USER_NAME" | grep -qw admins; then
   pass
else
   systemctl set-property --runtime "$UNIT" \
       CPUQuota=50% \
       MemoryMax=256M \
       >/dev/null 2>&1 || true
fi

exit 0
EOF

chmod +x /usr/local/bin/select-slice.sh

authselect create-profile limitedslice -b sssd &&

authselect select custom/limitedslice \
   with-fingerprint \
   with-silent-lastlog \
   with-mkhomedir &&

authselect apply-changes &&

sed -i \
   '/pam_systemd\.so/a session     optional     pam_exec.so seteuid /usr/local/bin/select-slice.sh' \
   /etc/authselect/custom/limitedslice/system-auth &&

sed -i \
   '/pam_systemd\.so/a session     optional     pam_exec.so seteuid /usr/local/bin/select-slice.sh' \
   /etc/authselect/custom/limitedslice/password-auth &&

authselect apply-changes
  • Configure SELinux permissions 
    semanage fcontext -a \
   -e /home \
   /trinity/home

semanage fcontext -a \
   -t httpd_sys_rw_content_t \
   '/home/[^/]+/ondemand(/.*)?'

restorecon -RF /trinity/home

semanage fcontext -a -t etc_t '/trinity/shared/etc/slurm/slurm.conf'

restorecon /trinity/shared/etc/slurm/slurm.conf
  • Configure Fail2Ban 
    cat > /etc/fail2ban/fail2ban.local << EOF
[Definition]
# Log levels: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG
loglevel = NOTICE
socket = /var/run/fail2ban/fail2ban.sock
pidfile = /var/run/fail2ban/fail2ban.pid
EOF

cat > /etc/fail2ban/jail.local << EOF
[sshd]
enabled = true
ignoreip = 127.0.0.1/8 ::1 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16
backend = systemd
maxretry = 10
bantime = 20h
findtime = 10m
EOF

fail2ban-client -t &&

systemctl restart fail2ban
  • Disable DNSSEC-validation 
    sed -i \
   's/^\([[:space:]]*\)\/\/[[:space:]]*\(dnssec-validation no;\)$/\1\2/' \
   /trinity/local/luna/daemon/templates/templ_dns_conf.cfg

sed -i \
   's/^\([[:space:]]*\)\/\/[[:space:]]*\(dnssec-validation no;\)$/\1\2/' \
   /etc/named.conf

named-checkconf /etc/named.conf &&

systemctl restart named
  • Export custom modules directory 
    mkdir -p /opt/modules

cat > /etc/profile.d/modules.sh << EOF
module use /opt/modules
module use /opt/shared_modules
EOF
  • Add an administrator to log into OOD portal 
    /usr/local/sbin/obol user add \
   --uid 1000 \
   --group admins \
   localadmin \
   -P

OOD Portal Customization

  • Download assets 
    cd /root

curl -LO https://nextcloud.bioeng.ru/s/jjkEtRcY5JFRBnb/download

unzip -qo download

rm -rf download assets

mv darwin_assets assets
  • Adjust page layouts and colors 
    tar -xzpf /root/assets/ood.tar.gz -C /

sed -i \
   -e '/^    <div class="nav-scroller shadow-md">/,/^    <\/div>$/d' \
   -e 's/Log in to Open OnDemand/Log in to Darwin/g' \
   -e 's|Log in to {{ issuer }}|Log in to Darwin|g' \
   /usr/share/ondemand-dex/web/templates/header.html

sed -i \
   -e '/^    <div class="row">$/,/^    <\/div>$/d' \
   -e '/^    {{ if "loginTitle" | extra }}/,/^    {{ end }}$/d' \
   -e '/^    {{ if "loginAlertMessage" | extra }}/,/^    {{ end }}$/d' \
   -e 's/Log in to Open OnDemand/Log in to Darwin/g' \
   -e 's|Log in to {{ issuer }}|Log in to Darwin|g' \
   /usr/share/ondemand-dex/web/templates/password.html

systemctl restart ondemand-dex

/bin/cp -f \
   /root/assets/icons/department.png \
   /var/www/ood/public/logo.png

/bin/cp -f \
   /root/assets/icons/favicon.ico \
   /var/www/ood/public/favicon.ico

/bin/cp -f \
   /root/assets/icons/department.png \
   /usr/share/ondemand-dex/web/themes/ondemand/favicon.png

rm -f \
   /usr/share/ondemand-dex/web/themes/ondemand/favicon.svg

find \
   /trinity/local/ondemand/3.0/ \
   -name favicon.ico \
   -exec /bin/cp /root/assets/icons/favicon.ico {} \;

find \
   /trinity/local/ondemand/3.0/ \
   -name logo.png \
   -exec /bin/cp /root/assets/icons/department.png {} \;
  • Install apps 
    tar -xzpf /root/assets/apps.tar.gz -C /

semodule -i ood_local.pp
  • Set up Grafana 
    tar -xzpf /root/assets/grafana.tar.gz -C /

systemctl daemon-reload

systemctl enable --now grafana-sync-authenticated-team.timer

setsebool -P grafana_can_tcp_connect_ldap_port on

semodule -i local_grafana_dbus.pp
  • Fix Luna API 
    sed -i \
   's/^  r\.headers_in\['\''X-Forwarded-Proto'\''\] = r\.is_https and "https" or "http"$/  if not r.headers_in['\''X-Forwarded-Proto'\''] or r.headers_in['\''X-Forwarded-Proto'\''] == '\'''\'' then\

    r.headers_in['\''X-Forwarded-Proto'\''] = r.is_https and "https" or "http"\

  end/' \
 /opt/ood/mod_ood_proxy/lib/ood/proxy.lua

Wiki Page and Web Proxy Creation

  • Disable ssl.conf and stop httpd from listening to port 80 
    sed -i \
 '/^Listen 80$/s/^/# /' \
 /etc/httpd/conf/httpd.conf
  • Install Certbot and create certificates 
    dnf install -y certbot python3-certbot-nginx

certbot --nginx \
   -d darwin.bioeng.ru \
   -d portal.darwin.bioeng.ru \
   -d cryosparc.darwin.bioeng.ru \
   -m <ADMIN_EMAIL> \
   --no-eff-email \
   --agree-tos \
   --redirect

OR 

tar -xzpf /root/assets/letsencrypt.tar.gz -C /
  • Set up NGINX 
    tar -xzpf /root/assets/nginx.tar.gz -C /

sed -i -e \
   '/dex_uri: \/dex/a\

logout_redirect: "/oidc?logout=https%3A%2F%2Fportal.darwin.bioeng.ru"\

oidc_settings:\
 OIDCRedirectURI: "https://portal.darwin.bioeng.ru/oidc"\
 OIDCProviderMetadataURL: "http://127.0.0.1:5556/dex/.well-known/openid-configuration"\
 OIDCXForwardedHeaders: "X-Forwarded-Proto X-Forwarded-Host X-Forwarded-Port"\
 OIDCSessionMaxDuration: 1209600 # 14 days \
 OIDCSessionInactivityTimeout: 43200 # 12 hours \
security_csp_frame_ancestors: "https://portal.darwin.bioeng.ru"' \
   -e '/^dex:$/a\
 client_redirect_uris:\
   - "https://portal.darwin.bioeng.ru/oidc"' \
   /etc/ood/config/ood_portal.yml &&

/opt/ood/ood-portal-generator/sbin/update_ood_portal &&

systemctl restart ondemand-dex httpd nginx

semanage fcontext -a -t httpd_log_t '/var/log/ondemand-nginx(/.*)?'

restorecon -RvF /var/log/ondemand-nginx
  • Fix reverse proxy to Grafana and Prometheus 
    sed -i \
   -e 's|--web.external-url=https://portal.darwin.bioeng.ru:9090/|--web.external-url=https://portal.darwin.bioeng.ru/prometheus/|' \
   /etc/systemd/system/prometheus-server.service &&

systemctl daemon-reload &&

systemctl restart prometheus-server &&

sed -i \
   -e 's|^root_url = http://darwin.cluster:3000$|root_url = https://portal.darwin.bioeng.ru/grafana/|' \
   /etc/grafana/grafana.ini
  • Install PHP 
    dnf install -y php php-fpm php-gd php-xml php-mbstring php-json &&

systemctl enable --now php-fpm
  • Install DokuWiki 
    tar -xzpf /root/assets/dokuwiki.tar.gz -C /

Compute Nodes Setup

  • Launch the compute node playbook 
    cd /root/trinityX/site

ansible-playbook compute-default.yml
  • Modify compute node image
    • Backup an image 
      luna osimage clone compute-image raw-compute-image-bak &&

tar -xzpf /root/assets/raid0.tar.gz -C /
    • Configure the image 
      lchroot compute-image
      • Add admins group to sudoers and allow them to refresh SSS cache 
        visudo

%admins  ALL=(ALL)  ALL
%admins ALL=(root) NOPASSWD: /usr/sbin/sss_cache -u *
      • Upgrade the kernel 
        dnf --refresh update 'kernel*'
      • Install packages 
        dnf config-manager --enable crb

dnf config-manager --add-repo \
   https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo

dnf install -y epel-release

dnf install -y \
   gh \
   nano \
   ripgrep \
   btop \
   fio \
   libfaketime \
   libreoffice \
   firefox \
   cabextract \
   nvidia-open \
   apptainer \
   https://github.com/VirtualGL/virtualgl/releases/download/3.1.4/VirtualGL-3.1.4.x86_64.rpm \
   https://github.com/NVIDIA/enroot/releases/download/v4.0.1/enroot-4.0.1-1.el8.x86_64.rpm \
   https://github.com/NVIDIA/enroot/releases/download/v4.0.1/enroot+caps-4.0.1-1.el8.x86_64.rpm
      • Make a scratch directory 
        mkdir /mnt/scratch

systemctl enable mount-scratch-raid.service
      • Set up shell initialization 
        cat > /etc/profile.d/conda.sh << EOF
if [ -f /opt/miniconda/etc/profile.d/conda.sh ]; then
   . /opt/miniconda/etc/profile.d/conda.sh
fi
EOF

cat > /etc/profile.d/modules.sh << EOF
module use /opt/modules
module use /opt/shared_modules
EOF

cat > /etc/profile.d/spack.sh << EOF
export SPACK_ROOT=/opt/spack
. /opt/spack/share/spack/setup-env.sh
module use \
   /opt/spack/share/spack/lmod/linux-rocky9-x86_64/Core
EOF
      • Disable pam_systemd (conflicts with pam_slurm_adopt) 
        BASE_PROFILE="$(authselect current --raw | awk '{print $1}')"

NEW_PROFILE='slurm-no-pam-systemd'

authselect create-profile \
   "$NEW_PROFILE" \
   -b "$BASE_PROFILE" \
   --symlink-meta &&

for f in \
   "/etc/authselect/custom/$NEW_PROFILE/password-auth" \
   "/etc/authselect/custom/$NEW_PROFILE/system-auth"
do
   sed -i \
       -e '/^[[:space:]]*-[[:space:]]*session[[:space:]]\+optional[[:space:]]\+pam_systemd\.so\([[:space:]].*\)\?$/d' \
       -e '/^[[:space:]]*session[[:space:]]\+optional[[:space:]]\+pam_systemd\.so\([[:space:]].*\)\?$/d' \
       "$f"
done

authselect select \
   custom/slurm-no-pam-systemd \
   with-mkhomedir \
   --force &&

authselect apply-changes
      • Add fonts 
        mkdir -p /usr/local/share/fonts/TimesNewRoman

curl -Lo /tmp/times32.exe \
   "https://sourceforge.net/projects/corefonts/files/the%20fonts/final/times32.exe"

cabextract /tmp/times32.exe -d /tmp/msfonts

cp \
   /tmp/msfonts/[Tt]imes*.[Tt][Tt][Ff] \
   /usr/local/share/fonts/TimesNewRoman

fc-cache -f

git clone \
   --depth 1 \
   --filter=blob:none \
   --sparse \
   https://github.com/google/fonts.git

cd fonts

git sparse-checkout set ofl/nunito

mkdir -p /usr/local/share/fonts/Nunito

cp ofl/nunito/*.ttf /usr/local/share/fonts/Nunito/

git sparse-checkout set ofl/manrope

mkdir -p /usr/local/share/fonts/Manrope

cp \
   ofl/manrope/Manrope\[wght\].ttf \
   /usr/local/share/fonts/Manrope/

fc-cache -f

cd .. && rm -rf fonts
      • Set up kernel dump and netconsole 
        cat > /etc/kdump.conf << 'EOKDUMP'
nfs 10.141.255.254:/var/crash/compute
path /
core_collector makedumpfile -l --message-level 7 -d 31
auto_reset_crashkernel yes
EOKDUMP

install \
   -d \
   -m 0755 \
   /etc/systemd/system \
   /etc/systemd/system/multi-user.target.wants

cat > /etc/systemd/system/netconsole-load.service << 'EOSVC'
[Unit]
Description=Load netconsole after network is up
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
# Cluster NIC name differs between nodes (eth0 or eth1), so derive it from
# the route to darwin (10.141.255.254) instead of hardcoding the device.
ExecStart=/bin/sh -c 'dev=$(ip -o route get 10.141.255.254 | sed -n "s/.* dev \\([^ ]*\\).*/\\1/p"); exec /usr/sbin/modprobe netconsole netconsole=6665@/$dev,6666@10.141.255.254/3c:ec:ef:0f:fb:5d'
ExecStop=/usr/sbin/modprobe -r netconsole
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOSVC

systemctl enable kdump.service

ln \
   -sfn \
   ../netconsole-load.service \
   /etc/systemd/system/multi-user.target.wants/netconsole-load.service
      • Set up the VGL server 
        cat > /etc/X11/Xwrapper.config << EOF
allowed_users=anybody
needs_root_rights=yes
EOF

vglserver_config
      • Exit the image 
        exit
    • Add crashkernel to cmdline 
      luna osimage change \
   -o \
   -qo 'net.ifnames=0 biosdevname=0 crashkernel=1G-2G:192M,2G-64G:256M,64G-:512M' \
   compute-image
    • Pack and backup the image 
      luna osimage pack compute-image &&

luna osimage clone compute-image full-compute-image-bak
  • Add IPMI network gateway 
    luna network change ipmi -g 10.148.255.254
  • Clean up node configurations 
    luna node remove node002

luna node remove node003

luna node remove node004

luna node rename node001 node01
  • Configure IB interface for 'node01' 
    luna node changeinterface node01 ib0 -N ib
  • Clone compute node 
    for i in $(seq -w 2 14); do
   luna node clone node01 node$i
done
  • Configure BMC interfaces for all compute nodes 
    for i in $(seq -w 1 14); do
   ip_octet=$(echo $i | sed 's/^0*//')
   luna node change node$i -if BMC -I 10.148.0.$ip_octet
done
  • Configure BMC 
    luna bmcsetup change compute-bmcsetup -u ADMIN -p <BMC_PASSWORD>
  • Reboot compute nodes and choose their node names manually in boot options 
    nodes=$(printf "node%02d," {1..14} | sed 's/,$//')

luna control power reset "$nodes"

Storage Node Setup

  • Clone the compute node image 
    luna osimage clone -o raw-compute-image-bak storage-image
  • Modify the image
    • Configure the image 
      lchroot storage-image
      • Remove unnecessary files 
        rm -rf /home/*

dnf remove slurm*

rm /etc/logrotate.d/slurm

sed -i \
   -e '/pam_slurm/d' \
   -e '/# Enable slurm-pam controls/d' \
   /etc/pam.d/sshd
      • Add admins group to sudoers and allow them to refresh SSS cache 
        visudo

%admins  ALL=(ALL)  ALL
%admins ALL=(root) NOPASSWD: /usr/sbin/sss_cache -u *
      • Upgrade the kernel 
        dnf --refresh update 'kernel*'
      • Install packages 
        dnf install -y \
   nano \
   ripgrep \
   btop \
   fio
      • Make a scratch directory 
        mkdir /mnt/scratch

systemctl enable mount-scratch-raid.service
      • Set up kernel dump and netconsole 
        cat > /etc/kdump.conf << 'EOKDUMP'
nfs 10.141.255.254:/var/crash/compute
path /
core_collector makedumpfile -l --message-level 7 -d 31
auto_reset_crashkernel yes
EOKDUMP

install \
   -d \
   -m 0755 \
   /etc/systemd/system \
   /etc/systemd/system/multi-user.target.wants

cat > /etc/systemd/system/netconsole-load.service << 'EOSVC'
[Unit]
Description=Load netconsole after network is up
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
# Cluster NIC name differs between nodes (eth0 or eth1), so derive it from
# the route to darwin (10.141.255.254) instead of hardcoding the device.
ExecStart=/bin/sh -c 'dev=$(ip -o route get 10.141.255.254 | sed -n "s/.* dev \\([^ ]*\\).*/\\1/p"); exec /usr/sbin/modprobe netconsole netconsole=6665@/$dev,6666@10.141.255.254/3c:ec:ef:0f:fb:5d'
ExecStop=/usr/sbin/modprobe -r netconsole
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOSVC

systemctl enable kdump.service

ln \
   -sfn \
   ../netconsole-load.service \
   /etc/systemd/system/multi-user.target.wants/netconsole-load.service
      • Exit the image 
        exit
    • Add crashkernel to cmdline 
      luna osimage change \
   -o \
   -qo 'net.ifnames=0 biosdevname=0 crashkernel=1G-2G:192M,2G-64G:256M,64G-:512M' \
   storage-image
    • Pack and backup the image 
      luna osimage pack storage-image &&

luna osimage clone storage-image full-storage-image-bak
  • Clone a compute node group 
    luna group clone compute-group storage-group
  • Clone a compute node 
    luna node clone node01 storage-node01
  • Clone a compute node to a storage node 
    luna node change storage-node01 -g storage-group
  • Set up the BMC IP for the storage node 
    luna node change storage-node01 -if BMC -I 10.148.1.1
  • Reboot the storage node and choose its name manually in boot options 
    luna control power reset storage-node01

Cluster Configuration

  • Configure shared folders 
    cd /opt

mkdir -p shared_modules shared_soft shared_envs

chmod 777 shared_modules shared_soft shared_envs

chmod +t shared_modules shared_soft shared_envs
  • Configure Slurm 
    tar -xzpf /root/assets/slurm.tar.gz -C /

systemctl daemon-reload

systemctl enable --now slurm-sync-users.timer

sacctmgr -i add \
   qos preemptible \
   set priority=1

sacctmgr -i add \
   qos normal \
   set priority=2

sacctmgr -i add \
   qos preempting \
   set \
       priority=3 \
       preempt=preemptible

sacctmgr -i add \
   account all_users\
   set \
       defaultqos=normal \
       qos=preemptible,normal,preempting

scontrol reconfigure
  • Add groups 
    /usr/local/sbin/obol group add intbio --gid 1002 &&

/usr/local/sbin/obol group add struct_biotech --gid 1053 &&

/usr/local/sbin/obol group add students --gid 1054 &&

/usr/local/sbin/obol group add guests --gid 1055 &&

/usr/local/sbin/obol group add carte_blanche --gid 1056 &&

/usr/local/sbin/obol group add ssh_allowed --gid 1057
  • Add users 
    /usr/local/sbin/obol user add testuser \
   --group students \
   --groups students,ssh_allowed \
   -P
  • Add admins group to sudoers 
    visudo

%admins  ALL=(ALL)  ALL
  • Allow only users in groups ssh_allowed, admins to connect to the head node via SSH 
    sed -i \
   "s/^\s*ldap_access_filter\s*=\s*.*$/ldap_access_filter = (|(memberOf=cn=admins,ou=group,dc=local)(memberOf=cn=ssh_allowed,ou=group,dc=local))/" \
   /etc/sssd/sssd.conf &&

systemctl restart sssd
  • Install Miniconda 
    curl -O https://repo.anaconda.com/miniconda/Miniconda3-py313_25.9.1-3-Linux-x86_64.sh &&

bash Miniconda3-py313_25.9.1-3-Linux-x86_64.sh \
   -b \
   -p /opt/miniconda &&

rm -rf Miniconda3-py313_25.9.1-3-Linux-x86_64.sh &&

cat > /etc/profile.d/conda.sh << 'EOF' &&
if [ -f /opt/miniconda/etc/profile.d/conda.sh ]; then
   . /opt/miniconda/etc/profile.d/conda.sh
fi

case "$-" in
 *i*) conda activate base >/dev/null 2>&1 ;;
esac
EOF

. /etc/profile.d/conda.sh &&

cat > /opt/miniconda/.condarc << EOF
envs_dirs:
 - ~/.conda/envs
 - /opt/shared_envs
EOF

conda tos accept \
   --override-channels \
   --channel https://repo.anaconda.com/pkgs/main &&

conda tos accept \
   --override-channels \
   --channel https://repo.anaconda.com/pkgs/r &&

conda update \
   -n base \
   -c main \
   conda \
   -y &&

conda create \
   -p /opt/miniconda/envs/authrevproxy \
   -c conda-forge \
   twisted \
   -y &&

conda create \
   -p /opt/miniconda/envs/tensorboard_app \
   -c conda-forge \
   tensorboard \
   -y

conda create \
   -p /opt/miniconda/envs/jupyter_app \
   -c conda-forge \
   jupyter nb_conda_kernels \
   jlab-enhanced-launcher jupyter-archive jupyterlab-h5web \
   jupyter-resource-usage \
   jupyter-lsp jupyterlab-lsp python-lsp-server \
   jupyterlab_code_formatter python-lsp-black black isort \
   nglview=3.1.4 "setuptools<80" \
   catppuccin-jupyterlab theme-darcula \
   -y &&

conda activate jupyter_app &&

pip install \
   jupyterlab_open_in_terminal_extension \
   JLDracula \
   jupyterlab-miami-nights \
   jupyterlab-theme-onedark \
   jupyterlab-theme-solarized-dark \
   jupyterlab-night \
   jupyterlab-midnightsea-theme \
   jupyterlab_ariakedark_theme \
   jupyterlab-horizon-theme \
   stellars-jupyterlab-darcula-theme \
   stellars-jupyterlab-sublime-theme \
   jupyterlab-theme-KULeuven \
   -y &&

jupyter labextension disable @jupyterlab/extensionmanager-extension
  • Spack
    • Install Spack 
      git clone \
   --depth 1 \
   --branch releases/latest \
   https://github.com/spack/spack \
   /opt/spack

chown -R root:root /opt/spack

find /opt/spack -type d -exec chmod 755 {} \;

find /opt/spack -type f -exec chmod 644 {} \;

chmod 755 /opt/spack/bin/spack &&

cat > /etc/profile.d/spack.sh << EOF
export SPACK_ROOT=/opt/spack
. /opt/spack/share/spack/setup-env.sh
module use /opt/spack/share/spack/lmod/linux-rocky9-x86_64/Core
EOF

. /etc/profile.d/spack.sh

spack config --scope site add modules:default:enable:[lmod]

spack config --scope site add \
   'modules:default:lmod:exclude_implicits:true'

spack config --scope site add modules:default:lmod:hash_length:0

spack config --scope site add \
   'modules:default:lmod:core_compilers:gcc@11.5.0'

spack config --scope site add 'packages:all:target:[x86_64]'
    • Install Cuda packages with max available minor version for every major version ≥ 10 
      spack install cuda@13.0.2 &&

spack install cuda@12.9.1 &&

spack install cuda@11.8.0 &&

spack install cuda@10.2.89
    • Install a recent code-server release (add a hash to `package.py` before executing) 
      spack checksum --add-to-package code-server@4.114.0

spack install code-server@4.114.0
    • Install OpenMPI 
      dnf install -y gcc-gfortran

spack install openmpi
    • Add your executables to Spack
  • Install Network UPS Tools
    • Install packages 
      dnf install -y \
   pkgconf-pkg-config \
   openssl-devel \
   libusbx-devel \
   net-snmp-devel \
   neon-devel \
   avahi-devel \
   systemd-devel \
   libtool-ltdl-devel
    • Download NUT 
      cd /usr/local/src

curl -LO https://www.networkupstools.org/source/2.8/nut-2.8.4.tar.gz

tar -xf nut-2.8.4.tar.gz
    • Configure NUT 
      cd nut-2.8.4

groupadd -r nut

useradd -r -g nut -s /sbin/nologin -d /var/lib/nut nut

./configure \
   --prefix=/usr/local \
   --sysconfdir=/etc/nut \
   --with-user=nut \
   --with-group=nut
    • Build NUT 
      make -j"$(nproc)"

make install

mkdir -p /var/state/ups /var/state/ups/upssched

chown nut:nut /var/state/ups /var/state/ups/upssched

chmod 770 /var/state/ups /var/state/ups/upssched

tar -xzpf /root/assets/nut.tar.gz -C /

printf '%s\n' '/usr/local/lib' > /etc/ld.so.conf.d/nut-local.conf

ldconfig
    • Download and set up Prometheus NUT exporter 
      curl -L \
   -o /tmp/nut_exporter-v3.2.5-linux-amd64 \
   https://github.com/DRuggeri/nut_exporter/releases/download/v3.2.5/nut_exporter-v3.2.5-linux-amd64

install -m 0755 \
   /tmp/nut_exporter-v3.2.5-linux-amd64 \
   /usr/local/bin/prometheus-nut-exporter

systemctl daemon-reload

systemctl enable --now nut-service prometheus-nut-exporter
  • Install CryoSPARC
    • Create an administrator 
      /usr/local/sbin/obol user add cryosparcuser \
   --group struct_biotech \
   -–home /trinity/structbio_home/cryosparcuser \
   -P
    • Download CryoSPARC 
      su cryosparcuser

LICENSE_ID="<CRYOSPARC_LICENSE_ID>"

curl -L \
   https://get.cryosparc.com/download/master-latest/$LICENSE_ID \
   -o cryosparc_master.tar.gz \
   --retry 5

curl -L \
   https://get.cryosparc.com/download/worker-latest/$LICENSE_ID \
   -o cryosparc_worker.tar.gz \
   --retry 5

tar -xf cryosparc_master.tar.gz cryosparc_master

tar -xf cryosparc_worker.tar.gz cryosparc_worker
    • Install master 
      cd cryosparc_master

./install.sh \
   --license $LICENSE_ID \
   --hostname $(hostname -f) \
   --dbpath /trinity/structbio_home/cryosparcuser/cryosparc_database \
   --port 39000 \
   --yes

source ~/.bashrc

cryosparcm start
    • Add first user 
      cryosparcm createuser \
   --email "<ADMIN_EMAIL>" \
   --username "<ADMIN_USERNAME>" \
   --firstname "<ADMIN_FIRST_NAME>" \
   --lastname "<ADMIN_LAST_NAME>"
    • Install worker 
      cd ../cryosparc_worker

./install.sh --license $LICENSE_ID --yes
    • Set up lanes 
      tar -xzpf /root/assets/cryosparc.tar.gz -C /

cd ../lanes/struct_biotech

cryosparcm cluster connect

cd ../intbio

cryosparcm cluster connect
    • Enable CryoSPARC service 
      systemctl enable cryosparc