Find the bad node before it stalls the run

SELECT s.hostname, p.name, p.version
FROM system_info s
JOIN deb_packages p
WHERE p.name LIKE 'cuda%'
   OR p.name LIKE '%nccl%';

SELECT s.hostname, k.name, k.status, k.used_by
FROM system_info s
JOIN kernel_modules k
WHERE k.name LIKE 'nvidia%';

SELECT s.hostname, p.pid, p.name, p.path,
       p.user_time, p.system_time
FROM system_info s
JOIN processes p
ORDER BY (p.user_time + p.system_time) DESC
LIMIT 25;

SELECT hostname, driver_version,
       cuda_version, nccl_version
FROM gpu_stack
WHERE driver_version != 'approved-version';

SELECT hostname, device, firmware_version
FROM accelerator_firmware
WHERE firmware_version != 'approved-version';

SELECT hostname, device, state, rate
FROM infiniband_ports
WHERE state != 'ACTIVE'
   OR rate != 'expected-rate';