administration_guides:darwin

Darwin Setup

Darwin Setup

This page documents the main steps used to set up Darwin.

Head Node OS Installation

Download Rocky Linux 9.8 x86_64 DVD ISO
Install ISO
Add password for root, unlock root, enable SSH with password
Add 'localadmin' administrator. DO NOT USE '!' OR '$' IN PASSWORD
Disable IPv6 on all network devices
Set hostname to 'darwin'
Merge '/' and '/home' (remove '/home' partition)

Head Node Setup

Connect to the head node
```
ssh localadmin@192.168.1.152

sudo -i
```

Enable persistent logging

systemd-tmpfiles --create --prefix /var/log/journal

mkdir -p /etc/systemd/journald.conf.d

printf '%s\n' '[Journal]' 'Storage=persistent' \
   >/etc/systemd/journald.conf.d/persistent.conf

systemctl restart systemd-journald

Set up kdump and netconsole

install \
   -d \
   -m 0755 \
   /var/log/netconsole \
   /var/crash/compute

tar -xzpf /root/assets/netconsole.tar.gz -C /

systemctl daemon-reload

cat > /etc/exports.d/kdump-compute.exports << 'EOF'
/var/crash/compute 10.141.0.0/16(rw,sync,no_subtree_check,no_root_squash)
EOF

exportfs -ra

systemctl reload nfs-server

systemctl restart rsyslog

Install the required packages

dnf install -y \
   epel-release \
   https://yum.osc.edu/ondemand/3.1/ondemand-release-compute-3.1-1.el9.noarch.rpm \
   https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm &&

dnf config-manager --set-enabled crb &&

dnf --enablerepo=elrepo-kernel install -y kmod-mlx4 &&

dnf install -y \
   git-core \
   gh \
   opensm \
   ripgrep \
   tmux \
   fio \
   iotop \
   selinux-policy-devel \
   fail2ban fail2ban-firewalld \
   turbovnc &&

systemctl enable --now opensm fail2ban &&

reboot

Disable GUI autoupdates

cat > /etc/dconf/db/local.d/00-gnome-software << EOF
[org/gnome/software]
allow-updates=false
download-updates=false
download-updates-notify=false
EOF

cat > /etc/dconf/db/local.d/locks/00-gnome-software << EOF
/org/gnome/software/allow-updates
/org/gnome/software/download-updates
/org/gnome/software/download-updates-notify
EOF

dconf update

Add network connections

nmcli con add \
   type ethernet \
   ifname eno2 \
   con-name internal \
   ipv4.method manual \
   ipv4.addresses 10.141.255.254/16 \
   ipv4.gateway "" \
   ipv6.method disable

nmcli con up internal

nmcli con add \
   type ethernet \
   ifname enp2s0 \
   con-name bmc \
   ipv4.method manual \
   ipv4.addresses 10.148.255.254/16 \
   ipv4.gateway "" \
   ipv6.method disable

nmcli con up bmc

nmcli con add \
   type infiniband \
   ifname ibp3s0 \
   con-name ib \
   ipv4.method manual \
   ipv4.addresses 10.149.255.254/16 \
   ipv4.gateway "" \
   ipv6.method disable

nmcli con up ib

Set up RAID

cat >> /etc/fstab/ << EOF
UUID=b5788192-0303-4ef5-944e-ac6354bd0938 /trinity/home ext4 defaults 0 0
UUID=fab54cdf-d1c6-4d7c-ae83-649a480ab3cb /trinity/structbio_home ext4 defaults,noatime 0 0
UUID=d27804ca-99df-4b46-ae65-1f04bfcb80fa /opt ext4 defaults 0 2
EOF

systemctl daemon-reload
cat > /etc/mdadm.conf << EOF
ARRAY /dev/md/newton:2 metadata=1.2 UUID=080b6df8:ddb3019f:665d341e:5fcaabc7
ARRAY /dev/md/struct_biotech metadata=1.2 spares=1 UUID=72c25cfc:fc70a47f:3d22ce14:7e5a00cc
EOF

If a proxy is required

Set environment variables

cat >> ~/.bashrc << 'EOF'
proxy() {
 local PROXY="http://localhost:10808"
 export http_proxy="$PROXY"
 export https_proxy="$PROXY"
 export HTTP_PROXY="$PROXY"
 export HTTPS_PROXY="$PROXY"
}
EOF

. ~/.bashrc

Forward a proxy port (use any free port if a VPN is enabled)

ssh -N -R 17890:127.0.0.1:<PORT> root@192.168.1.152

cat > /etc/profile.d/aliases.sh << 'EOF'
case "$-" in
 *i*) ;;
 *) return 0 2>/dev/null || exit 0 ;;
esac

alias vpn="ip route replace default via 192.168.1.182 dev eno1"
alias disvpn="ip route del default via 192.168.1.182 dev eno1"
EOF

chmod 700 /etc/profile.d/aliases.sh

Download trinityX

cd

git clone http://github.com/clustervision/trinityX &&

cd trinityX

Install trinityX

cd ~/trinityX

bash prepare.sh &&

cd site

Answer 'y' to a message about SELinux.

Reboot and repeat script run if asked to.

Answer 'n' to a message about ZFS.

Disable SSL

sed -i \
   -e 's/enable_ssl: true/enable_ssl: false/g' \
   controller.yml

sed -i \
   '/grafana_prometheus_url/s/:9090"/:9090\/prometheus"/' \
   /root/trinityX/site/controller.yml

Enable RDMA for NFS and mount /home, /shared, /opt, /ohpc

sed -i \
   "/^trix_ctrl_hostname:/a \

# -----------------------------------------------------------------------\
# Storage node IP address.\
# Used to mount shared filesystems (home, opt, shared, scratch) via NFS RDMA.\
# In a single-node setup, this should match the controller's InfiniBand IP.\
\
trix_storage_ip: 10.149.255.254" \
   group_vars/all.yml.example &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/# easybuild/,/disabled:/ {
       /disabled:/a \
       # opt ------------------------------------------------------------------------------ \
       - path: '\''/opt'\'' \
         options: '\''*(rw,async,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\'' \
       # structbio_home ------------------------------------------------------------------------------ \
       - path: '\''/trinity/structbio_home'\'' \
         options: '\''*(rw,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\'' \
    }' \
   controller.yml &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/remote:/s/{{ trix_ctrl_hostname }}/{{ trix_storage_ip }}/g' \
   -e "/- path: '{{ trix_shared }}'/,/options: 'defaults,nfsvers=4.2,ro,retrans=4,_netdev'/d" \
   -e "/- path: '{{ trix_home }}'/,/options: 'defaults,nfsvers=4.2,rw,retrans=4,noatime,_netdev'/d" \
   -e '1,/nfs_mounts:/!b' \
   -e '/nfs_mounts:/a \
   - path: '\''{{ trix_shared }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_shared }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,ro,retrans=4,noatime,_netdev'\''\
   - path: '\''{{ trix_home }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_home }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/trinity/structbio_home'\''\
     remote: '\''{{ trix_storage_ip }}:/trinity/structbio_home'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/opt'\''\
     remote: '\''{{ trix_storage_ip }}:/opt'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\'''\
   imports/trinity-redhat-image-setup.yml

Enable PAM Slurm adopt module

sed -i \
   -e '/^account[[:space:]]\{1,\}include[[:space:]]\{1,\}password-auth$/a\account    required     pam_slurm_adopt.so' \
   /root/trinityX/site/roles/trinity/slurm/files/pam-sshd

sed -i \
   's/^enable_slurm_pam:[[:space:]]*false$/enable_slurm_pam: true/' \
   /root/trinityX/site/roles/trinity/slurm/defaults/main.yml

Fix Prometheus path

sed -i \
   $'/when: luna2_daemon_pip is undefined/a\

- name: Patch luna2 daemon Prometheus route prefix\
     ansible.builtin.replace:\
   path: '\''{{ luna2_python_dir.stdout }}/site-packages/daemon/plugins/import/prometheus_hw_rules.py'\''\
   regexp: '\''self\\.prometheus_url = "https://localhost:9090"'\''\
   replace: '\''self.prometheus_url = "http://localhost:9090/prometheus"'\''' \
   /root/trinityX/site/roles/trinity/luna2/tasks/controller.yml

Prepare Ansible inventory

cat > hosts << EOF
[controllers]
darwin ansible_connection=local
EOF

Prepare the head node playbook

cp group_vars/all.yml.example group_vars/all.yml

sed -i \
   -e 's/^trix_ctrl1_hostname:.*/trix_ctrl1_hostname: darwin/' \
   -e "s|^trix_external_fqdn:.*|trix_external_fqdn: 'portal.darwin.bioeng.ru'|" \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.8\.8"/\1- "192.168.1.3"/' \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.4\.4"/\1- "192.168.1.1"/' \
   -e 's/^luna_password:.*/luna_password: .../' \
   -e 's/^luna_protocol:.*/luna_protocol: http/' \
   -e 's/^firewalld_public_interfaces:.*/firewalld_public_interfaces: [eno1]/' \
   -e 's/^firewalld_trusted_interfaces:.*/firewalld_trusted_interfaces: [eno2, enp23s0, ibp180s0]/' \
   -e 's/^firewalld_public_tcp_ports:.*/firewalld_public_tcp_ports: [22, 80, 443]/' \
   group_vars/all.yml

Patch playbook to install OOD apps idempotently

sed -i -f - /root/trinityX/site/roles/trinity/ood-apps/tasks/app.yml <<'SED'
/^    - name: Make folder for app /,/^      notify: Run update_ood$/ {
 /^      notify: Run update_ood$/ a\

   - name: Ensure web server can traverse trinity app directory\
     ansible.posix.acl:\
       path: "{{ app_path }}"\
       entity: apache\
       etype: user\
       permissions: rx\
       state: present\
     when:\
       - app_enabled\
       - app.get('type') == 'trinity'\
     notify: Run update_ood
}
SED

Launch the head node playbook
```
ansible-playbook controller.yml
```

Enable RDMA for NFS and mount /home, /shared, /opt, /ohpc

sed -i \
   "/^trix_ctrl_hostname:/a \\

# -----------------------------------------------------------------------\
# Storage node IP address.\
# Used to mount shared filesystems (home, opt, shared, scratch) via NFS RDMA.\
# In a single-node setup, this should match the controller's InfiniBand IP.\
\
trix_storage_ip: 10.149.255.254" \
   group_vars/all.yml.example &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/# easybuild/,/disabled:/ {
       /disabled:/a \
       # opt ------------------------------------------------------------------------------ \
       - path: '\''/opt'\'' \
         options: '\''*(rw,async,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\'' \
       # structbio_home ------------------------------------------------------------------------------ \
       - path: '\''/trinity/structbio_home'\'' \
         options: '\''*(rw,no_root_squash)'\'' \
         rwhosts: '\''{{ all_ctrl_hostname | join(" ") }} {{ trix_ctrl_hostname if ha else "" }}'\''
    }' \
   controller.yml &&

sed -i \
   -e 's/nfs_enable_rdma: false/nfs_enable_rdma: true/' \
   -e '/remote:/s/{{ trix_ctrl_hostname }}/{{ trix_storage_ip }}/g' \
   -e "/- path: '{{ trix_shared }}'/,/options: 'defaults,nfsvers=4.2,ro,retrans=4,_netdev'/d" \
   -e "/- path: '{{ trix_home }}'/,/options: 'defaults,nfsvers=4.2,rw,retrans=4,noatime,_netdev'/d" \
   -e '1,/nfs_mounts:/!b' \
   -e '/nfs_mounts:/a \
   - path: '\''{{ trix_shared }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_shared }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,ro,retrans=4,noatime,_netdev'\''\
   - path: '\''{{ trix_home }}'\''\
     remote: '\''{{ trix_storage_ip }}:{{ trix_home }}'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/trinity/structbio_home'\''\
     remote: '\''{{ trix_storage_ip }}:/trinity/structbio_home'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\''\
   - path: '\''/opt'\''\
     remote: '\''{{ trix_storage_ip }}:/opt'\''\
     options: '\''defaults,nfsvers=4.2,rsize=1048576,wsize=1048576,hard,timeo=600,rw,retrans=4,noatime,_netdev'\'''\
   imports/trinity-redhat-image-setup.yml

Enable PAM Slurm adopt module

sed -i \
   -e '/^account[[:space:]]\{1,\}include[[:space:]]\{1,\}password-auth$/a\account    required     pam_slurm_adopt.so' \
   /root/trinityX/site/roles/trinity/slurm/files/pam-sshd

sed -i \
   's/^enable_slurm_pam:[[:space:]]*false$/enable_slurm_pam: true/' \
   /root/trinityX/site/roles/trinity/slurm/defaults/main.yml

Fix Prometheus path

sed -i \
   $'/when: luna2_daemon_pip is undefined/a\

- name: Patch luna2 daemon Prometheus route prefix\
     ansible.builtin.replace:\
   path: '\''{{ luna2_python_dir.stdout }}/site-packages/daemon/plugins/import/prometheus_hw_rules.py'\''\
   regexp: '\''self\\.prometheus_url = "https://localhost:9090"'\''\
   replace: '\''self.prometheus_url = "http://localhost:9090/prometheus"'\''' \
   /root/trinityX/site/roles/trinity/luna2/tasks/controller.yml

Prepare Ansible inventory

cat > hosts << EOF
[controllers]
darwin ansible_connection=local
EOF

Prepare the head node playbook

cp group_vars/all.yml.example group_vars/all.yml

sed -i \
   -e 's/^trix_ctrl1_hostname:.*/trix_ctrl1_hostname: darwin/' \
   -e "s|^trix_external_fqdn:.*|trix_external_fqdn: '\''portal.darwin.bioeng.ru'\''|" \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.8\.8"/\1- "192.168.1.3"/' \
   -e 's/^\([[:space:]]*\)-[[:space:]]*"8\.8\.4\.4"/\1- "192.168.1.1"/' \
   -e 's/^luna_password:.*/luna_password: hKpT5APf@jfp5g/' \
   -e 's/^luna_protocol:.*/luna_protocol: http/' \
   -e 's/^firewalld_public_interfaces:.*/firewalld_public_interfaces: [eno1]/' \
   -e 's/^firewalld_trusted_interfaces:.*/firewalld_trusted_interfaces: [eno2, enp23s0, ibp180s0]/' \
   -e 's/^firewalld_public_tcp_ports:.*/firewalld_public_tcp_ports: [22, 80, 443]/' \
   group_vars/all.yml

Patch playbook to install OOD apps idempotently

sed -i -f - /root/trinityX/site/roles/trinity/ood-apps/tasks/app.yml <<'SED'
/^    - name: Make folder for app /,/^      notify: Run update_ood$/ {
 /^      notify: Run update_ood$/ a\

   - name: Ensure web server can traverse trinity app directory\
     ansible.posix.acl:\
       path: "{{ app_path }}"\
       entity: apache\
       etype: user\
       permissions: rx\
       state: present\
     when:\
       - app_enabled\
       - app.get('type') == 'trinity'\
     notify: Run update_ood
}
SED

Launch the head node playbook
```
ansible-playbook controller.yml
```

Limit non-admin users on the head node by 0.5 CPU and 256 MB RAM

cat > /usr/local/bin/select-slice.sh << EOF
#!/bin/bash

USER_NAME="${PAM_USER:-}"
[ -z "$USER_NAME" ] && exit 0

UID_NUM="$(id -u "$USER_NAME" 2>/dev/null)" || exit 0

export DBUS_SYSTEM_BUS_ADDRESS="unix:path=/run/dbus/system_bus_socket"
UNIT="user-${UID_NUM}.slice"

if id -nG "$USER_NAME" | grep -qw admins; then
   pass
else
   systemctl set-property --runtime "$UNIT" \
       CPUQuota=50% \
       MemoryMax=256M \
       >/dev/null 2>&1 || true
fi

exit 0
EOF

chmod +x /usr/local/bin/select-slice.sh

authselect create-profile limitedslice -b sssd &&

authselect select custom/limitedslice \
   with-fingerprint \
   with-silent-lastlog \
   with-mkhomedir &&

authselect apply-changes &&

sed -i \
   '/pam_systemd\.so/a session     optional     pam_exec.so seteuid /usr/local/bin/select-slice.sh' \
   /etc/authselect/custom/limitedslice/system-auth &&

sed -i \
   '/pam_systemd\.so/a session     optional     pam_exec.so seteuid /usr/local/bin/select-slice.sh' \
   /etc/authselect/custom/limitedslice/password-auth &&

authselect apply-changes

Configure SELinux permissions

semanage fcontext -a \
   -e /home \
   /trinity/home

semanage fcontext -a \
   -t httpd_sys_rw_content_t \
   '/home/[^/]+/ondemand(/.*)?'

restorecon -RF /trinity/home

semanage fcontext -a -t etc_t '/trinity/shared/etc/slurm/slurm.conf'

restorecon /trinity/shared/etc/slurm/slurm.conf

Configure Fail2Ban

cat > /etc/fail2ban/fail2ban.local << EOF
[Definition]
# Log levels: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG
loglevel = NOTICE
socket = /var/run/fail2ban/fail2ban.sock
pidfile = /var/run/fail2ban/fail2ban.pid
EOF

cat > /etc/fail2ban/jail.local << EOF
[sshd]
enabled = true
ignoreip = 127.0.0.1/8 ::1 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16
backend = systemd
maxretry = 10
bantime = 20h
findtime = 10m
EOF

fail2ban-client -t &&

systemctl restart fail2ban

Disable DNSSEC-validation

sed -i \
   's/^\([[:space:]]*\)\/\/[[:space:]]*\(dnssec-validation no;\)$/\1\2/' \
   /trinity/local/luna/daemon/templates/templ_dns_conf.cfg

sed -i \
   's/^\([[:space:]]*\)\/\/[[:space:]]*\(dnssec-validation no;\)$/\1\2/' \
   /etc/named.conf

named-checkconf /etc/named.conf &&

systemctl restart named

Export custom modules directory

mkdir -p /opt/modules

cat > /etc/profile.d/modules.sh << EOF
module use /opt/modules
module use /opt/shared_modules
EOF

Add an administrator to log into OOD portal

/usr/local/sbin/obol user add \
   --uid 1000 \
   --group admins \
   localadmin \
   -P

OOD Portal Customization

Download assets

cd /root

curl -LO https://nextcloud.bioeng.ru/s/jjkEtRcY5JFRBnb/download

unzip -qo download

rm -rf download assets

mv darwin_assets assets

Adjust page layouts and colors

tar -xzpf /root/assets/ood.tar.gz -C /

sed -i \
   -e '/^    <div class="nav-scroller shadow-md">/,/^    <\/div>$/d' \
   -e 's/Log in to Open OnDemand/Log in to Darwin/g' \
   -e 's|Log in to {{ issuer }}|Log in to Darwin|g' \
   /usr/share/ondemand-dex/web/templates/header.html

sed -i \
   -e '/^    <div class="row">$/,/^    <\/div>$/d' \
   -e '/^    {{ if "loginTitle" | extra }}/,/^    {{ end }}$/d' \
   -e '/^    {{ if "loginAlertMessage" | extra }}/,/^    {{ end }}$/d' \
   -e 's/Log in to Open OnDemand/Log in to Darwin/g' \
   -e 's|Log in to {{ issuer }}|Log in to Darwin|g' \
   /usr/share/ondemand-dex/web/templates/password.html

systemctl restart ondemand-dex

/bin/cp -f \
   /root/assets/icons/department.png \
   /var/www/ood/public/logo.png

/bin/cp -f \
   /root/assets/icons/favicon.ico \
   /var/www/ood/public/favicon.ico

/bin/cp -f \
   /root/assets/icons/department.png \
   /usr/share/ondemand-dex/web/themes/ondemand/favicon.png

rm -f \
   /usr/share/ondemand-dex/web/themes/ondemand/favicon.svg

find \
   /trinity/local/ondemand/3.0/ \
   -name favicon.ico \
   -exec /bin/cp /root/assets/icons/favicon.ico {} \;

find \
   /trinity/local/ondemand/3.0/ \
   -name logo.png \
   -exec /bin/cp /root/assets/icons/department.png {} \;

Install apps

tar -xzpf /root/assets/apps.tar.gz -C /

semodule -i ood_local.pp

Set up Grafana

tar -xzpf /root/assets/grafana.tar.gz -C /

systemctl daemon-reload

systemctl enable --now grafana-sync-authenticated-team.timer

setsebool -P grafana_can_tcp_connect_ldap_port on

semodule -i local_grafana_dbus.pp

Fix Luna API

sed -i \
   's/^  r\.headers_in\['\''X-Forwarded-Proto'\''\] = r\.is_https and "https" or "http"$/  if not r.headers_in['\''X-Forwarded-Proto'\''] or r.headers_in['\''X-Forwarded-Proto'\''] == '\'''\'' then\

    r.headers_in['\''X-Forwarded-Proto'\''] = r.is_https and "https" or "http"\

  end/' \
 /opt/ood/mod_ood_proxy/lib/ood/proxy.lua

Wiki Page and Web Proxy Creation

Disable ssl.conf and stop httpd from listening to port 80

sed -i \
 '/^Listen 80$/s/^/# /' \
 /etc/httpd/conf/httpd.conf

Install Certbot and create certificates

dnf install -y certbot python3-certbot-nginx

certbot --nginx \
   -d darwin.bioeng.ru \
   -d portal.darwin.bioeng.ru \
   -d cryosparc.darwin.bioeng.ru \
   -m <ADMIN_EMAIL> \
   --no-eff-email \
   --agree-tos \
   --redirect

tar -xzpf /root/assets/letsencrypt.tar.gz -C /

Set up NGINX

tar -xzpf /root/assets/nginx.tar.gz -C /

sed -i -e \
   '/dex_uri: \/dex/a\

logout_redirect: "/oidc?logout=https%3A%2F%2Fportal.darwin.bioeng.ru"\

oidc_settings:\
 OIDCRedirectURI: "https://portal.darwin.bioeng.ru/oidc"\
 OIDCProviderMetadataURL: "http://127.0.0.1:5556/dex/.well-known/openid-configuration"\
 OIDCXForwardedHeaders: "X-Forwarded-Proto X-Forwarded-Host X-Forwarded-Port"\
 OIDCSessionMaxDuration: 1209600 # 14 days \
 OIDCSessionInactivityTimeout: 43200 # 12 hours \
security_csp_frame_ancestors: "https://portal.darwin.bioeng.ru"' \
   -e '/^dex:$/a\
 client_redirect_uris:\
   - "https://portal.darwin.bioeng.ru/oidc"' \
   /etc/ood/config/ood_portal.yml &&

/opt/ood/ood-portal-generator/sbin/update_ood_portal &&

systemctl restart ondemand-dex httpd nginx

semanage fcontext -a -t httpd_log_t '/var/log/ondemand-nginx(/.*)?'

restorecon -RvF /var/log/ondemand-nginx

Fix reverse proxy to Grafana and Prometheus

sed -i \
   -e 's|--web.external-url=https://portal.darwin.bioeng.ru:9090/|--web.external-url=https://portal.darwin.bioeng.ru/prometheus/|' \
   /etc/systemd/system/prometheus-server.service &&

systemctl daemon-reload &&

systemctl restart prometheus-server &&

sed -i \
   -e 's|^root_url = http://darwin.cluster:3000$|root_url = https://portal.darwin.bioeng.ru/grafana/|' \
   /etc/grafana/grafana.ini

Install PHP

dnf install -y php php-fpm php-gd php-xml php-mbstring php-json &&

systemctl enable --now php-fpm

Install DokuWiki

tar -xzpf /root/assets/dokuwiki.tar.gz -C /

Compute Nodes Setup

Launch the compute node playbook

cd /root/trinityX/site

ansible-playbook compute-default.yml

Modify compute node image

Backup an image

luna osimage clone compute-image raw-compute-image-bak &&

tar -xzpf /root/assets/raid0.tar.gz -C /

Configure the image

lchroot compute-image

Add admins group to sudoers and allow them to refresh SSS cache

visudo

%admins  ALL=(ALL)  ALL
%admins ALL=(root) NOPASSWD: /usr/sbin/sss_cache -u *

Upgrade the kernel
```
dnf --refresh update 'kernel*'
```

Install packages

dnf config-manager --enable crb

dnf config-manager --add-repo \
   https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo

dnf install -y epel-release

dnf install -y \
   gh \
   nano \
   ripgrep \
   btop \
   fio \
   libfaketime \
   libreoffice \
   firefox \
   cabextract \
   nvidia-open \
   apptainer \
   https://github.com/VirtualGL/virtualgl/releases/download/3.1.4/VirtualGL-3.1.4.x86_64.rpm \
   https://github.com/NVIDIA/enroot/releases/download/v4.0.1/enroot-4.0.1-1.el8.x86_64.rpm \
   https://github.com/NVIDIA/enroot/releases/download/v4.0.1/enroot+caps-4.0.1-1.el8.x86_64.rpm

Make a scratch directory

mkdir /mnt/scratch

systemctl enable mount-scratch-raid.service

Set up shell initialization

cat > /etc/profile.d/conda.sh << EOF
if [ -f /opt/miniconda/etc/profile.d/conda.sh ]; then
   . /opt/miniconda/etc/profile.d/conda.sh
fi
EOF

cat > /etc/profile.d/modules.sh << EOF
module use /opt/modules
module use /opt/shared_modules
EOF

cat > /etc/profile.d/spack.sh << EOF
export SPACK_ROOT=/opt/spack
. /opt/spack/share/spack/setup-env.sh
module use \
   /opt/spack/share/spack/lmod/linux-rocky9-x86_64/Core
EOF

Disable pam_systemd (conflicts with pam_slurm_adopt)

BASE_PROFILE="$(authselect current --raw | awk '{print $1}')"

NEW_PROFILE='slurm-no-pam-systemd'

authselect create-profile \
   "$NEW_PROFILE" \
   -b "$BASE_PROFILE" \
   --symlink-meta &&

for f in \
   "/etc/authselect/custom/$NEW_PROFILE/password-auth" \
   "/etc/authselect/custom/$NEW_PROFILE/system-auth"
do
   sed -i \
       -e '/^[[:space:]]*-[[:space:]]*session[[:space:]]\+optional[[:space:]]\+pam_systemd\.so\([[:space:]].*\)\?$/d' \
       -e '/^[[:space:]]*session[[:space:]]\+optional[[:space:]]\+pam_systemd\.so\([[:space:]].*\)\?$/d' \
       "$f"
done

authselect select \
   custom/slurm-no-pam-systemd \
   with-mkhomedir \
   --force &&

authselect apply-changes

Add fonts

mkdir -p /usr/local/share/fonts/TimesNewRoman

curl -Lo /tmp/times32.exe \
   "https://sourceforge.net/projects/corefonts/files/the%20fonts/final/times32.exe"

cabextract /tmp/times32.exe -d /tmp/msfonts

cp \
   /tmp/msfonts/[Tt]imes*.[Tt][Tt][Ff] \
   /usr/local/share/fonts/TimesNewRoman

fc-cache -f

git clone \
   --depth 1 \
   --filter=blob:none \
   --sparse \
   https://github.com/google/fonts.git

cd fonts

git sparse-checkout set ofl/nunito

mkdir -p /usr/local/share/fonts/Nunito

cp ofl/nunito/*.ttf /usr/local/share/fonts/Nunito/

git sparse-checkout set ofl/manrope

mkdir -p /usr/local/share/fonts/Manrope

cp \
   ofl/manrope/Manrope\[wght\].ttf \
   /usr/local/share/fonts/Manrope/

fc-cache -f

cd .. && rm -rf fonts

Set up kernel dump and netconsole

cat > /etc/kdump.conf << 'EOKDUMP'
nfs 10.141.255.254:/var/crash/compute
path /
core_collector makedumpfile -l --message-level 7 -d 31
auto_reset_crashkernel yes
EOKDUMP

install \
   -d \
   -m 0755 \
   /etc/systemd/system \
   /etc/systemd/system/multi-user.target.wants

cat > /etc/systemd/system/netconsole-load.service << 'EOSVC'
[Unit]
Description=Load netconsole after network is up
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
# Cluster NIC name differs between nodes (eth0 or eth1), so derive it from
# the route to darwin (10.141.255.254) instead of hardcoding the device.
ExecStart=/bin/sh -c 'dev=$(ip -o route get 10.141.255.254 | sed -n "s/.* dev \\([^ ]*\\).*/\\1/p"); exec /usr/sbin/modprobe netconsole netconsole=6665@/$dev,6666@10.141.255.254/3c:ec:ef:0f:fb:5d'
ExecStop=/usr/sbin/modprobe -r netconsole
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOSVC

systemctl enable kdump.service

ln \
   -sfn \
   ../netconsole-load.service \
   /etc/systemd/system/multi-user.target.wants/netconsole-load.service

Set up the VGL server

cat > /etc/X11/Xwrapper.config << EOF
allowed_users=anybody
needs_root_rights=yes
EOF

vglserver_config

Exit the image
```
exit
```

Add crashkernel to cmdline

luna osimage change \
   -o \
   -qo 'net.ifnames=0 biosdevname=0 crashkernel=1G-2G:192M,2G-64G:256M,64G-:512M' \
   compute-image

Pack and backup the image

luna osimage pack compute-image &&

luna osimage clone compute-image full-compute-image-bak

Add IPMI network gateway

luna network change ipmi -g 10.148.255.254

Clean up node configurations

luna node remove node002

luna node remove node003

luna node remove node004

luna node rename node001 node01

Configure IB interface for 'node01'

luna node changeinterface node01 ib0 -N ib

Clone compute node

for i in $(seq -w 2 14); do
   luna node clone node01 node$i
done

Configure BMC interfaces for all compute nodes

for i in $(seq -w 1 14); do
   ip_octet=$(echo $i | sed 's/^0*//')
   luna node change node$i -if BMC -I 10.148.0.$ip_octet
done

Configure BMC

luna bmcsetup change compute-bmcsetup -u ADMIN -p <BMC_PASSWORD>

Reboot compute nodes and choose their node names manually in boot options

nodes=$(printf "node%02d," {1..14} | sed 's/,$//')

luna control power reset "$nodes"

Storage Node Setup

Clone the compute node image

luna osimage clone -o raw-compute-image-bak storage-image

Modify the image

Configure the image

lchroot storage-image

Remove unnecessary files

rm -rf /home/*

dnf remove slurm*

rm /etc/logrotate.d/slurm

sed -i \
   -e '/pam_slurm/d' \
   -e '/# Enable slurm-pam controls/d' \
   /etc/pam.d/sshd

Add admins group to sudoers and allow them to refresh SSS cache

visudo

%admins  ALL=(ALL)  ALL
%admins ALL=(root) NOPASSWD: /usr/sbin/sss_cache -u *

Upgrade the kernel
```
dnf --refresh update 'kernel*'
```

Install packages

dnf install -y \
   nano \
   ripgrep \
   btop \
   fio

Make a scratch directory

mkdir /mnt/scratch

systemctl enable mount-scratch-raid.service

Set up kernel dump and netconsole

cat > /etc/kdump.conf << 'EOKDUMP'
nfs 10.141.255.254:/var/crash/compute
path /
core_collector makedumpfile -l --message-level 7 -d 31
auto_reset_crashkernel yes
EOKDUMP

install \
   -d \
   -m 0755 \
   /etc/systemd/system \
   /etc/systemd/system/multi-user.target.wants

cat > /etc/systemd/system/netconsole-load.service << 'EOSVC'
[Unit]
Description=Load netconsole after network is up
Wants=network-online.target
After=network-online.target

[Service]
Type=oneshot
# Cluster NIC name differs between nodes (eth0 or eth1), so derive it from
# the route to darwin (10.141.255.254) instead of hardcoding the device.
ExecStart=/bin/sh -c 'dev=$(ip -o route get 10.141.255.254 | sed -n "s/.* dev \\([^ ]*\\).*/\\1/p"); exec /usr/sbin/modprobe netconsole netconsole=6665@/$dev,6666@10.141.255.254/3c:ec:ef:0f:fb:5d'
ExecStop=/usr/sbin/modprobe -r netconsole
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOSVC

systemctl enable kdump.service

ln \
   -sfn \
   ../netconsole-load.service \
   /etc/systemd/system/multi-user.target.wants/netconsole-load.service

Exit the image
```
exit
```

Add crashkernel to cmdline

luna osimage change \
   -o \
   -qo 'net.ifnames=0 biosdevname=0 crashkernel=1G-2G:192M,2G-64G:256M,64G-:512M' \
   storage-image

Pack and backup the image

luna osimage pack storage-image &&

luna osimage clone storage-image full-storage-image-bak

Clone a compute node group

luna group clone compute-group storage-group

Clone a compute node
```
luna node clone node01 storage-node01
```

Clone a compute node to a storage node

luna node change storage-node01 -g storage-group

Set up the BMC IP for the storage node

luna node change storage-node01 -if BMC -I 10.148.1.1

Reboot the storage node and choose its name manually in boot options
```
luna control power reset storage-node01
```

Cluster Configuration

Configure shared folders

cd /opt

mkdir -p shared_modules shared_soft shared_envs

chmod 777 shared_modules shared_soft shared_envs

chmod +t shared_modules shared_soft shared_envs

Configure Slurm

tar -xzpf /root/assets/slurm.tar.gz -C /

systemctl daemon-reload

systemctl enable --now slurm-sync-users.timer

sacctmgr -i add \
   qos preemptible \
   set priority=1

sacctmgr -i add \
   qos normal \
   set priority=2

sacctmgr -i add \
   qos preempting \
   set \
       priority=3 \
       preempt=preemptible

sacctmgr -i add \
   account all_users\
   set \
       defaultqos=normal \
       qos=preemptible,normal,preempting

scontrol reconfigure

Add groups

/usr/local/sbin/obol group add intbio --gid 1002 &&

/usr/local/sbin/obol group add struct_biotech --gid 1053 &&

/usr/local/sbin/obol group add students --gid 1054 &&

/usr/local/sbin/obol group add guests --gid 1055 &&

/usr/local/sbin/obol group add carte_blanche --gid 1056 &&

/usr/local/sbin/obol group add ssh_allowed --gid 1057

Add users

/usr/local/sbin/obol user add testuser \
   --group students \
   --groups students,ssh_allowed \
   -P

Add admins group to sudoers
```
visudo

%admins  ALL=(ALL)  ALL
```

Allow only users in groups ssh_allowed, admins to connect to the head node via SSH

sed -i \
   "s/^\s*ldap_access_filter\s*=\s*.*$/ldap_access_filter = (|(memberOf=cn=admins,ou=group,dc=local)(memberOf=cn=ssh_allowed,ou=group,dc=local))/" \
   /etc/sssd/sssd.conf &&

systemctl restart sssd

Install Miniconda

curl -O https://repo.anaconda.com/miniconda/Miniconda3-py313_25.9.1-3-Linux-x86_64.sh &&

bash Miniconda3-py313_25.9.1-3-Linux-x86_64.sh \
   -b \
   -p /opt/miniconda &&

rm -rf Miniconda3-py313_25.9.1-3-Linux-x86_64.sh &&

cat > /etc/profile.d/conda.sh << 'EOF' &&
if [ -f /opt/miniconda/etc/profile.d/conda.sh ]; then
   . /opt/miniconda/etc/profile.d/conda.sh
fi

case "$-" in
 *i*) conda activate base >/dev/null 2>&1 ;;
esac
EOF

. /etc/profile.d/conda.sh &&

cat > /opt/miniconda/.condarc << EOF
envs_dirs:
 - ~/.conda/envs
 - /opt/shared_envs
EOF

conda tos accept \
   --override-channels \
   --channel https://repo.anaconda.com/pkgs/main &&

conda tos accept \
   --override-channels \
   --channel https://repo.anaconda.com/pkgs/r &&

conda update \
   -n base \
   -c main \
   conda \
   -y &&

conda create \
   -p /opt/miniconda/envs/authrevproxy \
   -c conda-forge \
   twisted \
   -y &&

conda create \
   -p /opt/miniconda/envs/tensorboard_app \
   -c conda-forge \
   tensorboard \
   -y

conda create \
   -p /opt/miniconda/envs/jupyter_app \
   -c conda-forge \
   jupyter nb_conda_kernels \
   jlab-enhanced-launcher jupyter-archive jupyterlab-h5web \
   jupyter-resource-usage \
   jupyter-lsp jupyterlab-lsp python-lsp-server \
   jupyterlab_code_formatter python-lsp-black black isort \
   nglview=3.1.4 "setuptools<80" \
   catppuccin-jupyterlab theme-darcula \
   -y &&

conda activate jupyter_app &&

pip install \
   jupyterlab_open_in_terminal_extension \
   JLDracula \
   jupyterlab-miami-nights \
   jupyterlab-theme-onedark \
   jupyterlab-theme-solarized-dark \
   jupyterlab-night \
   jupyterlab-midnightsea-theme \
   jupyterlab_ariakedark_theme \
   jupyterlab-horizon-theme \
   stellars-jupyterlab-darcula-theme \
   stellars-jupyterlab-sublime-theme \
   jupyterlab-theme-KULeuven \
   -y &&

jupyter labextension disable @jupyterlab/extensionmanager-extension

Spack

Install Spack

git clone \
   --depth 1 \
   --branch releases/latest \
   https://github.com/spack/spack \
   /opt/spack

chown -R root:root /opt/spack

find /opt/spack -type d -exec chmod 755 {} \;

find /opt/spack -type f -exec chmod 644 {} \;

chmod 755 /opt/spack/bin/spack &&

cat > /etc/profile.d/spack.sh << EOF
export SPACK_ROOT=/opt/spack
. /opt/spack/share/spack/setup-env.sh
module use /opt/spack/share/spack/lmod/linux-rocky9-x86_64/Core
EOF

. /etc/profile.d/spack.sh

spack config --scope site add modules:default:enable:[lmod]

spack config --scope site add \
   'modules:default:lmod:exclude_implicits:true'

spack config --scope site add modules:default:lmod:hash_length:0

spack config --scope site add \
   'modules:default:lmod:core_compilers:gcc@11.5.0'

spack config --scope site add 'packages:all:target:[x86_64]'

Install Cuda packages with max available minor version for every major version ≥ 10

spack install cuda@13.0.2 &&

spack install cuda@12.9.1 &&

spack install cuda@11.8.0 &&

spack install cuda@10.2.89

Install a recent code-server release (add a hash to `package.py` before executing)

spack checksum --add-to-package code-server@4.114.0

spack install code-server@4.114.0

Install OpenMPI

dnf install -y gcc-gfortran

spack install openmpi

Add your executables to Spack

Install Network UPS Tools

Install packages

dnf install -y \
   pkgconf-pkg-config \
   openssl-devel \
   libusbx-devel \
   net-snmp-devel \
   neon-devel \
   avahi-devel \
   systemd-devel \
   libtool-ltdl-devel

Download NUT

cd /usr/local/src

curl -LO https://www.networkupstools.org/source/2.8/nut-2.8.4.tar.gz

tar -xf nut-2.8.4.tar.gz

Configure NUT

cd nut-2.8.4

groupadd -r nut

useradd -r -g nut -s /sbin/nologin -d /var/lib/nut nut

./configure \
   --prefix=/usr/local \
   --sysconfdir=/etc/nut \
   --with-user=nut \
   --with-group=nut

Build NUT

make -j"$(nproc)"

make install

mkdir -p /var/state/ups /var/state/ups/upssched

chown nut:nut /var/state/ups /var/state/ups/upssched

chmod 770 /var/state/ups /var/state/ups/upssched

tar -xzpf /root/assets/nut.tar.gz -C /

printf '%s\n' '/usr/local/lib' > /etc/ld.so.conf.d/nut-local.conf

ldconfig

Download and set up Prometheus NUT exporter

curl -L \
   -o /tmp/nut_exporter-v3.2.5-linux-amd64 \
   https://github.com/DRuggeri/nut_exporter/releases/download/v3.2.5/nut_exporter-v3.2.5-linux-amd64

install -m 0755 \
   /tmp/nut_exporter-v3.2.5-linux-amd64 \
   /usr/local/bin/prometheus-nut-exporter

systemctl daemon-reload

systemctl enable --now nut-service prometheus-nut-exporter

Install CryoSPARC

Create an administrator

/usr/local/sbin/obol user add cryosparcuser \
   --group struct_biotech \
   -–home /trinity/structbio_home/cryosparcuser \
   -P

Download CryoSPARC

su cryosparcuser

LICENSE_ID="<CRYOSPARC_LICENSE_ID>"

curl -L \
   https://get.cryosparc.com/download/master-latest/$LICENSE_ID \
   -o cryosparc_master.tar.gz \
   --retry 5

curl -L \
   https://get.cryosparc.com/download/worker-latest/$LICENSE_ID \
   -o cryosparc_worker.tar.gz \
   --retry 5

tar -xf cryosparc_master.tar.gz cryosparc_master

tar -xf cryosparc_worker.tar.gz cryosparc_worker

Install master

cd cryosparc_master

./install.sh \
   --license $LICENSE_ID \
   --hostname $(hostname -f) \
   --dbpath /trinity/structbio_home/cryosparcuser/cryosparc_database \
   --port 39000 \
   --yes

source ~/.bashrc

cryosparcm start

Add first user

cryosparcm createuser \
   --email "<ADMIN_EMAIL>" \
   --username "<ADMIN_USERNAME>" \
   --firstname "<ADMIN_FIRST_NAME>" \
   --lastname "<ADMIN_LAST_NAME>"

Install worker

cd ../cryosparc_worker

./install.sh --license $LICENSE_ID --yes

Set up lanes

tar -xzpf /root/assets/cryosparc.tar.gz -C /

cd ../lanes/struct_biotech

cryosparcm cluster connect

cd ../intbio

cryosparcm cluster connect

Enable CryoSPARC service
```
systemctl enable cryosparc
```

Table of Contents

Darwin Setup

Head Node OS Installation

Head Node Setup

OOD Portal Customization

Wiki Page and Web Proxy Creation

Compute Nodes Setup

Storage Node Setup

Cluster Configuration