#!/usr/share/venvs/netq-agent/bin/python
#
#  SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#  SPDX-License-Identifier: LicenseRef-NvidiaProprietary
#
#  NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
#  property and proprietary rights in and to this material, related
#  documentation and any modifications thereto. Any use, reproduction,
#  disclosure or distribution of this material and related documentation
#  without an express license agreement from NVIDIA CORPORATION or
#  its affiliates is strictly prohibited.
#

import json
import subprocess
import os
import sys
import socket
import argparse
from time import localtime, sleep
from netq_lib.common.utils import subprocess_wrapper, is_sonic, is_cl3, is_cl4, is_bcm
from concurrent import futures
import multiprocessing


DATE = '%d%d%d' % (localtime().tm_year, localtime().tm_mon, localtime().tm_mday)
TIME = '%d%d%d' % (localtime().tm_hour, localtime().tm_min, localtime().tm_sec)

SUPPORT_PATH = '/var/support/'
EXTENSION = '.txz' if not is_sonic() else '.tar.gz'
TAR_OPTIONS = '-cJvf' if not is_sonic() else '-czvf'
OPTAPREFIX = 'opta'
NETQPREFIX = 'netq'
BASENAME = 'support_%s_%s_%s' % (socket.gethostname(), DATE, TIME)
NETQ_BASENAME = '%s_%s_%s' % (socket.gethostname(), DATE, TIME)
BASENAME_COMBINED = 'support_combined_%s_%s_%s' % (socket.gethostname(), DATE, TIME)
UNIFIED_BASENAME = 'unified_support_%s_%s_%s' % (socket.gethostname(), DATE, TIME)
NETQ_AGENT_PY3_4_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.4/site-packages/netq_agent'
NETQ_AGENT_PY3_6_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.6/site-packages/netq_agent'
NETQ_AGENT_PY3_7_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.7/site-packages/netq_agent'
NETQ_AGENT_PY3_8_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.8/site-packages/netq_agent'
NETQ_AGENT_PY3_10_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.10/site-packages/netq_agent'
NETQ_AGENT_PY3_11_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.11/site-packages/netq_agent'
NETQ_AGENT_PY3_12_VENV_PATH = '/usr/share/venvs/netq-agent/lib/python3.12/site-packages/netq_agent'
NETQ_CLI_CLIENT_PATH = '/tests/netq_agent/agent_cli_client.py'
NETQ_AGENT_VENV_PATH = ''
NETQ_CLIENT_CLI = ''
NETQ_CMD_SCRIPT = 'netq_script.sh'
OPTA_FILE = '/etc/app-release'
#COPY_DB_LOGS = 'db_logs'
MASTER_IP_FILE = '/mnt/admin/master_ip'
MASTER_IPV6_FILE = '/mnt/admin/master_ipv6'
WORKER_IP_FILE = '/mnt/admin/worker_ip'
SSH_DIR = '/mnt/admin/ssh'
MASTER_LOCAL_SUPPORT_PATH = '/var/support/'

RC_SUCCESS = 0
RC_FAIL = 1

CMD_TIMEOUT = 60
FAILED_COMMANDS_FILE = 'failed-commands.txt'
KAFKA_CONNECT_SERVICE_IP_COMMAND = ['kubectl', 'get', 'service/kafka-connect', '-n', 'netq-infra', '-o', 'jsonpath={.spec.clusterIP}']
KAFKA_CONNECT_SERVICE_IP = ''

OPTA_FILES = {
    '/var/log/': ['syslog', 'auth', 'netq-agent', 'netqd', 'boot', 'kern', 'containers', 'dpkg', 'kubelet', 'apt'],
    '/var/log/netq/': ['fio-disk-io-test.json'],
    '/etc/': ['os-release', 'app-release', 'resolv.conf', 'netq', 'kubernetes', 'hosts', 'netplan'],
    '/home/nvidia/': ['kube'],
    '/var/run/': ['netq'],
    '/usr/sbin/': ['opta-support-utils'],
    '/mnt/': ['netq-heap-dump'],
}

PODS_FILES = {
    'dc1-default-sts': 'var/log/cassandra'
}

CUMULUS_FILES = {
    '/var/log/': ['syslog', 'netq-agent', 'netqd', 'netq-model', 'boot', 'dpkg', 'auth'],
    '/etc/': ['os-release', 'netq'],
    '/var/run/': ['netq'],
}

OPTA_CMDS = {
    'kubectl-describe.txt': ['kubectl', 'describe', 'pods', '--all-namespaces'],
    'kubectl-getall.txt': ['kubectl', 'get', 'all', '--all-namespaces'],
    'kubectl-describe-node.txt': ['kubectl', 'describe', 'node'],
    'kubectl-top-node.txt': ['kubectl', 'top', 'nodes'],
    'kubectl-top-pod.txt': ['kubectl', 'top', 'pods', '--all-namespaces'],
    'kubectl-get-netqkafka.txt': ['kubectl', 'get', 'netkafka', '-n', 'netq-infra', '-o', 'yaml'],
    'kubectl-get-netqinstaller.txt': ['kubectl', 'get', 'netqinstaller', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqedge.txt': ['kubectl', 'get', 'netqedge', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqclustermanager.txt': ['kubectl', 'get', 'netqclustermanager', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqcentral.txt': ['kubectl', 'get', 'netqcentral', '-n', 'netq-eth', '-o', 'yaml'],
    'kubectl-get-netqapps.txt': ['kubectl', 'get', 'netqapps', '-n', 'netq-eth', '-o', 'yaml'],
    'kubelet-pod-volume-usage.txt': ['bash', '-c', 'du -sh /var/lib/kubelet/pods/*/volumes/*/* 2>/dev/null | sort -hr'],
    'prometheus-volume-stats.txt': [
    'bash', '-c', r'''
PROM_IP=$(kubectl get svc prometheus-k8s -n monitoring -o jsonpath="{.spec.clusterIP}") && \
curl -G "http://$PROM_IP:9090/api/v1/query" \
  --data-urlencode "query=sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes) / 1024 / 1024 / 1024" \
  | jq -r ".data.result[] | \"\\(.metric.namespace)\t\\(.metric.persistentvolumeclaim)\t\\(.value[1]) GB\"" \
  | column -t
'''
],
    "vmstorage-disk-usage.txt": [
        "bash",
        "-c",
        "kubectl -n netq-infra get pods -o name 2>/dev/null | grep vmstorage-victoria-metrics "
        "| while read pod; do kubectl -n netq-infra exec $pod -- sh -c \"echo -n '$pod '; df -h /var/log | awk 'NR==2 {print \\$3, \\$4, \\$5}'\" 2>/dev/null; done"
    ],
    'crictl-images.txt': ['crictl', 'image', 'ls'],
    'crictl-info.txt': ['crictl', 'info'],
    'crictl-ps.txt': ['crictl', 'ps', '-a'],
    'opta-check.txt': ['opta-check'],
    'netq-show-agents.txt': ['netq', 'show', 'agents'],
    'netq-show-lldp.txt': ['netq', 'show', 'lldp'],
    'netq-show-inv-os.txt': ['netq', 'show', 'inventory', 'os'],
    'netq-show-clag.txt': ['netq', 'show', 'clag'],
    'netq-show-bgp.txt': ['netq', 'show', 'bgp'],
    'netq-show-evpn.txt': ['netq', 'show', 'evpn'],
    'netq-show-ospf.txt': ['netq', 'show', 'ospf'],
    'netq-show-opta-health.txt': ['netq', 'show', 'opta-health'],
    'netq-show-status.txt': ['netq', 'show', 'status', 'verbose'],
    'netq-show-events.txt': ['netq', 'show', 'events', 'between', 'now', 'and', '24h'],
    'netq-show-otlp-endpoints.txt': ['netq', 'show', 'otlp', 'endpoints'],
    'netq-show-otlp-health-verbose.txt': ['netq', 'show', 'otlp', 'health', 'verbose'],
    'netq-show-otlp-security-mode.txt': ['netq', 'show', 'otlp', 'security-mode'],
    'netq-show-otlp-tls-cert.txt': ['netq', 'show', 'otlp', 'tls-cert'],
    'netq-show-otlp-tls-ca-cert.txt': ['netq', 'show', 'otlp', 'tls-ca-cert'],
    'netq-app-admin.txt': "crictl ps -aq label=io.kubernetes.container.name=netq-app-admin",
    'topology-json.txt': ['topology-json'],
    'dmesg.txt': ['dmesg'],
    'lspci.txt': ['lspci'],
    'ls-mnt-r.txt': ['ls', '-lR', '/mnt/'],
    'chrony.txt': ['chronyc', 'tracking'],
    'kubectl-describe-configmap.txt': ['kubectl', 'describe', 'configmap', '--all-namespaces'],
    'kubectl-describe-ingress.txt': ['kubectl', 'describe', 'ingress', '--all-namespaces'],
    'kubectl-describe-httproute.txt': ['kubectl', 'describe', 'httproute', '--all-namespaces'],
    'kubectl-describe-grpcroute.txt': ['kubectl', 'describe', 'grpcroute', '--all-namespaces'],
    'kubectl-describe-gateway.txt': ['kubectl', 'describe', 'gateway', '--all-namespaces'],
    'kubectl-describe-clusterrole.txt': ['kubectl', 'describe', 'clusterrole'],
    'kubectl-describe-role.txt': ['kubectl', 'describe', 'role', '--all-namespaces'],
    'kubectl-describe-serviceaccount.txt': ['kubectl', 'describe', 'serviceaccount', '--all-namespaces'],
    'kubectl-get-serviceaccount.txt': ['kubectl', 'get', 'serviceaccount', '--all-namespaces'],
    'kubectl-get-role.txt': ['kubectl', 'get', 'role', '--all-namespaces'],
    'kubectl-get-clusterrole.txt': ['kubectl', 'get', 'clusterrole'],
    'kubectl-get-crd.txt': ['kubectl', 'get', 'crd', '--all-namespaces'],
    'kubectl-get-secret.txt': ['kubectl', 'get', 'secret', '--all-namespaces'],
    'kubectl-get-configmap.txt': ['kubectl', 'get', 'configmap', '--all-namespaces'],
    'kubectl-get-ingress.txt': ['kubectl', 'get', 'ingress', '--all-namespaces'],
    'kubectl-get-httproute.txt': ['kubectl', 'get', 'httproute', '--all-namespaces'],
    'kubectl-get-grpcroute.txt': ['kubectl', 'get', 'grpcroute', '--all-namespaces'],
    'kubectl-get-gateway.txt': ['kubectl', 'get', 'gateway', '--all-namespaces'],
    'kubectl-get-gatewayclass.txt': ['kubectl', 'get', 'gatewayclass'],
    'kafka-connector-status.txt': ['curl', 'http://{service_ip}:8083/connectors?expand=status'],
    'opta-info.txt': ['/usr/sbin/opta-info.py'],
    'cassandra-tables-disk-usage.txt': ['du', '-h', '/mnt/cassandra'],
    'cassandra-describe-cluster.txt': ['bash', '/usr/sbin/netq-cassandra-information-dump.sh'],
    'netqd-file-descriptors.txt': ['sh', '-c', 'true'],
    'kubectl-get-strimzi.txt': ['kubectl', 'get', 'kafkas,kafkaconnects,kafkatopics,kafkausers', '--all-namespaces', '-o', 'wide'],
    'kubectl-describe-strimzi.txt': ['kubectl', 'describe', 'kafkas,kafkaconnects', '--all-namespaces'],
    'kubectl-get-connectors.txt': ['kubectl', 'get', 'kafkaconnectors.kafka.strimzi.io', '--all-namespaces', '-o', 'wide'],
    'kubectl-describe-connectors.txt': ['kubectl', 'describe', 'kafkaconnectors.kafka.strimzi.io', '--all-namespaces'],
    'kafka-consumer-group-list.txt': ['/usr/sbin/netq-kafka-consumer-list.sh'],
    'kafka-consumer-group-lag.txt': ['/usr/sbin/netq-kafka-consumer-lag.sh'],
    'sysstat-sar-decoded.txt': "find /var/log/sa /var/log/sysstat -maxdepth 1 -name 'sa[0-9]*' -type f 2>/dev/null | sort | while read f; do echo \"===== SAR data from $f (10 min interval) =====\"; sar -A -i 600 -f \"$f\" 2>/dev/null; done",
}

LINUX_CMDS = {
    'df-h.txt': ['df', '-h'],
    'lsmod.txt': ['lsmod'],
    'top-n1.txt': ['top', '-b', '-n1'],
    'ss-pan.txt': ['ss', '-pan'],
    'free-lm.txt': ['free', '-lm'],
    'ps-aux.txt': ['ps', 'aux'],
    'systemctl.txt': ['systemctl', '-l', '--type', 'service', '--all'],
    'ip-link-show.txt': ['ip', '-d', 'link', 'show'],
    'ip-addr-show.txt': ['ip', 'addr', 'show'],
    'ip-route-show-table-all.txt': ['ip', 'route', 'show', 'table', 'all'],
    'ip-neighbor-show.txt': ['ip', 'neighbor', 'show'],
    'brctl-show.txt': ['brctl', 'show'],
    'bridge-fdb-show.txt': ['bridge', 'fdb', 'show'],
    'bridge-vlan-show.txt': ['bridge', 'vlan', 'show'],
    'ntpq.txt': ['ntpq', '-pn'],
    'timedatectl.txt': ['timedatectl'],
    'netq-agent-cli.txt': ['python', NETQ_CLIENT_CLI],
    'proc_meminfo.txt': ['cat', '/proc/meminfo'],
    'proc_cpuinfo.txt': ['cat', '/proc/cpuinfo'],
    'journalctl_containerd.txt': ['journalctl', '--no-pager', '-u', 'containerd'],
    'env_nvidia.txt': ['sudo', 'su', '-l', 'nvidia', '-c', 'printenv'],
    'env_root.txt': ['sudo', 'su', '-l', 'root', '-c', 'printenv'],
    'ansible_log.txt': ['cat', '/mnt/admin/tmp/ansible.log'],
    'ulimit.txt': ['bash', '-c', 'ulimit -a'],
    'threads-max.txt': ['cat', '/proc/sys/kernel/threads-max'],
    'vm_backuprestore_log.txt': ['cat', '/var/log/vm-backuprestore.log'],
    'netqd-file-descriptors.txt': ['sh', '-c', 'true'],
    'sysstat-sar-decoded.txt': "find /var/log/sa /var/log/sysstat -maxdepth 1 -name 'sa[0-9]*' -type f 2>/dev/null | sort | while read f; do echo \"===== SAR data from $f (10 min interval) =====\"; sar -A -i 600 -f \"$f\" 2>/dev/null; done",
}

DEBIAN_UBUNTU_FILES = {
    '/var/log/': ['syslog', 'netq-agent', 'netqd', 'netq-model'],
    '/etc/': ['os-release', 'netq'],
    '/var/run/': ['netq'],
}
DEBIAN_UBUNTU_CMDS = {
    'dpkg.txt': ['dpkg', '-l'],
}
CENTOS_RHEL_FILES = {
    '/var/log/': ['messages', 'netq-agent', 'netqd', 'netq-model'],
    '/etc/': ['os-release', 'netq'],
    '/var/run/': ['netq'],
}
CENTOS_RHEL_CMDS = {
    'rpm-qa.txt': ['rpm', '-qa']
}

SONIC_FILES = {
    '/var/log/': ['syslog', 'netq-agent', 'netqd', 'netq-model', 'boot', 'dpkg', 'auth'],
    '/etc/': ['os-release', 'netq'],
    '/etc/sonic/': ['sonic_version.yml'],
    '/var/run/': ['netq'],
}

SONIC_CMDS = {
    'config-db-json.txt': ['/usr/local/bin/sonic-cfggen', '-d', '--print-data'],
}

def get_safe_xz_threads():
    try:
        total_cores = multiprocessing.cpu_count()
        # Use 50% of cores if available and not more than 8 for xz compression
        safe_threads = min(8, max(1, int(total_cores * 0.5)))
        return safe_threads
    except Exception:
        return 2

XZ_THREADS = get_safe_xz_threads()

def get_netqd_fd_count():
    """Get file descriptor count for netqd process using lsof."""
    try:
        # Find netqd process PID
        ps_cmd = ['ps', 'aux']
        result = subprocess.run(ps_cmd, capture_output=True, text=True, timeout=10)
        if result.returncode != 0:
            return "Error: Could not get process list"

        netqd_pid = None
        for line in result.stdout.split('\n'):
            if 'netqd' in line and 'grep' not in line:
                parts = line.split()
                if len(parts) > 1:
                    netqd_pid = parts[1]
                    break

        if not netqd_pid:
            return "netqd process not found"

        # Get file descriptor count using lsof
        lsof_cmd = ['lsof', '-p', netqd_pid]
        lsof_result = subprocess.run(lsof_cmd, capture_output=True, text=True, timeout=30)

        if lsof_result.returncode != 0:
            return f"Error: lsof failed for PID {netqd_pid} - {lsof_result.stderr}"

        # Count lines (each line represents an open file descriptor)
        fd_count = len(lsof_result.stdout.strip().split('\n')) - 1  # Subtract header line

        # Also get system limits for context
        try:
            import resource
            soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
            return f"netqd PID: {netqd_pid}\nFile descriptors in use: {fd_count}\nSystem soft limit: {soft_limit}\nSystem hard limit: {hard_limit}\n\nDetailed lsof output:\n{lsof_result.stdout}"
        except ImportError:
            return f"netqd PID: {netqd_pid}\nFile descriptors in use: {fd_count}\n\nDetailed lsof output:\n{lsof_result.stdout}"

    except subprocess.TimeoutExpired:
        return "Error: Command timed out while getting file descriptor information"
    except Exception as e:
        return f"Error: {str(e)}"

def is_opta():
    return os.path.isfile(OPTA_FILE)


def is_cumulus():
    return (is_cl3() or is_cl4())


def platform():
    sysname, nodename, release, version, machine = os.uname()

    if os.path.isdir(NETQ_AGENT_PY3_4_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_4_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_6_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_6_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_7_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_7_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_8_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_8_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_10_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_10_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_11_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_11_VENV_PATH
    elif os.path.isdir(NETQ_AGENT_PY3_12_VENV_PATH):
        NETQ_AGENT_VENV_PATH = NETQ_AGENT_PY3_12_VENV_PATH
    else:
        NETQ_AGENT_VENV_PATH = ''
        major = sys.version_info.major
        minor = sys.version_info.minor
        print('skipping model dump - python version {}.{} - not supported'.format(major, minor))

    if len(NETQ_AGENT_VENV_PATH):
        NETQ_CLIENT_CLI = '%s%s' % (NETQ_AGENT_VENV_PATH, NETQ_CLI_CLIENT_PATH)
        LINUX_CMDS.update({'netq-agent-cli.txt': ['python3', NETQ_CLIENT_CLI]})

    if is_opta():
        OPTA_CMDS.update(LINUX_CMDS)
        OPTA_CMDS.update(DEBIAN_UBUNTU_CMDS)
        return OPTA_FILES, OPTA_CMDS
    elif 'Cumulus' in version:
        LINUX_CMDS.update(DEBIAN_UBUNTU_CMDS)
        return CUMULUS_FILES, LINUX_CMDS
    elif 'Ubuntu' in version or 'Debian' in version:
        DEBIAN_UBUNTU_CMDS.update(LINUX_CMDS)
        if is_sonic():
            DEBIAN_UBUNTU_CMDS.update(SONIC_CMDS)
            return SONIC_FILES, DEBIAN_UBUNTU_CMDS
        return DEBIAN_UBUNTU_FILES, DEBIAN_UBUNTU_CMDS
    elif 'el7' in release:
        try:
            # brctl is not installed by default
            subprocess_wrapper(['yum', 'install', 'bridge-utils.x86_64', '-y'])
        except subprocess.CalledProcessError as ex:
            print('ERROR: {}'.format(ex.cmd))
        CENTOS_RHEL_CMDS.update(LINUX_CMDS)
        return CENTOS_RHEL_FILES, CENTOS_RHEL_CMDS

    else: # rhel
        try:
            # brctl is not installed by default
            subprocess_wrapper(['yum', 'install', 'bridge-utils.x86_64', '-y'])
        except subprocess.CalledProcessError as ex:
            print('ERROR: {}'.format(ex.cmd))
        CENTOS_RHEL_CMDS.update(LINUX_CMDS)
        return CENTOS_RHEL_FILES, CENTOS_RHEL_CMDS


def copy_files(file_dict, support_path):
    cp_files = []
    for path in file_dict:
        if not os.path.exists(path):
            continue
        for file in os.listdir(path):
            for file_pattern in file_dict[path]:
                if file_pattern in file:
                    cp_files.append('%s%s' % (path, file))

    try:
        for file in cp_files:
            # Avoid dereferencing symlinks in /var/log/containers to prevent cp failures on broken links
            if file == '/var/log/containers' or file.startswith('/var/log/containers/'):
                subprocess_wrapper(['cp', '-r', '-L', '--parents', file, support_path + '/Logs'])
            else:
                subprocess_wrapper(['cp', '-r', '-L', '--parents', file, support_path + '/Logs'])
    except subprocess.CalledProcessError as ex:
        print('ERROR: failed to {}'.format(ex.cmd))
        return RC_FAIL
    else:
        return RC_SUCCESS


def copy_pods_files(support_path):
    try:
        for pod_name in PODS_FILES:
            copy_pod_files(pod_name, PODS_FILES[pod_name], support_path)
    except Exception as ex:
        print('WARNING: failed to {}'.format(ex.cmd))
    return RC_SUCCESS


def copy_pod_files(pod_name, directory_path, support_path):
    """
    Copy files from pods. For cassandra logs, copies individual files to avoid kubectl cp timeout issues.
    """
    full_pod_names = get_full_pod_name(pod_name)
    for full_pod_name in full_pod_names:
        namespace, pod_name_only = full_pod_name.split('/', 1)
        dest_dir = f'{support_path}/Logs/{full_pod_name}'
        
        # For cassandra logs, copy individual files to avoid kubectl cp directory timeout issues
        if 'cassandra' in directory_path:
            print(f'Collecting cassandra logs from {full_pod_name}...')
            
            # Create destination directory
            subprocess.run(['mkdir', '-p', dest_dir], check=False)
            
            # Get list of log files from the pod
            try:
                list_cmd = f'kubectl exec -n {namespace} {pod_name_only} -- ls /{directory_path}/ 2>/dev/null'
                result = subprocess.run(list_cmd, shell=True, capture_output=True, text=True, timeout=30)
                if result.returncode == 0:
                    log_files = result.stdout.strip().split('\n')
                    print(f'  Found {len(log_files)} log file(s) to copy')
                    
                    # Copy each log file individually
                    success_count = 0
                    for log_file in log_files:
                        if not log_file.strip():
                            continue
                        
                        source_path = f'{directory_path}/{log_file}'
                        dest_path = f'{dest_dir}/{log_file}'
                        kubectl_cp_cmd = f'kubectl cp -n {namespace} {pod_name_only}:{source_path} {dest_path} 2>/dev/null'
                        
                        try:
                            cp_result = subprocess.run(kubectl_cp_cmd, shell=True, capture_output=True, timeout=60)
                            if cp_result.returncode == 0:
                                success_count += 1
                            else:
                                print(f'  Warning: Failed to copy {log_file}')
                        except subprocess.TimeoutExpired:
                            print(f'  Warning: Timeout copying {log_file}')
                        except Exception as ex:
                            print(f'  Warning: Error copying {log_file}: {str(ex)}')
                    
                    print(f'  Successfully copied {success_count}/{len(log_files)} log file(s)')
                else:
                    print(f'  Warning: Could not list files in {directory_path}')
            except Exception as ex:
                print(f'  Error listing cassandra log files: {str(ex)}')
        else:
            # Original logic for non-cassandra pods
            kubectl_exec_cassandra_command = f'kubectl cp -n {namespace} {pod_name_only}:{directory_path} {dest_dir} 2>/dev/null'
            timeout = 10*60
            try:
                shell_process = subprocess.Popen(kubectl_exec_cassandra_command, stdin=subprocess.PIPE,
                        stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
                shell_process.wait(timeout)
            except subprocess.TimeoutExpired:
                print(f'WARNING: kubectl cp timed out after {timeout} seconds for {full_pod_name}')
                shell_process.kill()
                shell_process.wait()
            except Exception as ex:
                print(f'ERROR: kubectl cp failed for {full_pod_name}: {str(ex)}')


def get_full_pod_name(pod_name):
    kubectl_get_pod_command = f"kubectl get pods --all-namespaces 2>/dev/null | grep {pod_name}"
    output = subprocess.check_output(kubectl_get_pod_command, shell=True, stderr=subprocess.DEVNULL)
    lines = output.decode("utf-8").split("\n")
    full_name_pods = []
    for line in lines:
        if pod_name in line:
            parts = line.split()
            if len(parts) >= 2:
                namespace = parts[0]
                pod_name_full = parts[1]
                full_name_pods.append(f"{namespace}/{pod_name_full}")
    return full_name_pods


def collect_var_log_pods(support_path, namespaces=None):
    """
    Copy complete /var/log/pods directories for specified namespaces.
    This captures all rotated logs (0.log.1, 0.log.2, etc.) and previous container instances.

    Args:
        support_path: Base path for support bundle
        namespaces: List of namespaces to collect (default: ['netq-eth'])
    """
    if namespaces is None:
        namespaces = ['netq-eth']
    
    var_log_pods_path = '/var/log/pods'

    print(f'Collecting complete pod logs from {var_log_pods_path} for namespaces: {", ".join(namespaces)}...')

    if not os.path.exists(var_log_pods_path):
        print(f'  WARNING: {var_log_pods_path} does not exist')
        return

    if not os.access(var_log_pods_path, os.R_OK):
        print(f'  WARNING: {var_log_pods_path} is not readable (permission denied)')
        return

    dest_base = f'{support_path}/Logs/var/log/pods'
    try:
        subprocess_wrapper(['mkdir', '-p', dest_base])
    except Exception as e:
        print(f'  ERROR: Failed to create destination directory {dest_base}: {e}')
        return

    # List all pod directories once (not namespace-specific)
    try:
        pod_dirs = os.listdir(var_log_pods_path)
    except PermissionError as e:
        print(f'  ERROR: Permission denied listing {var_log_pods_path}: {str(e)}')
        return
    except Exception as e:
        print(f'  ERROR: Failed to list {var_log_pods_path}: {str(e)}')
        return

    # Copy pod directories for each namespace
    for namespace in namespaces:
        # Filter for the specified namespace
        namespace_prefix = f'{namespace}_'
        matching_pods = [d for d in pod_dirs if d.startswith(namespace_prefix)]

        if not matching_pods:
            continue

        # Copy each matching pod directory
        for pod_dir in matching_pods:
            source_path = os.path.join(var_log_pods_path, pod_dir)
            dest_path = os.path.join(dest_base, pod_dir)

            # Verify source is a directory before copying
            if not os.path.isdir(source_path):
                print(f'  WARNING: {source_path} is not a directory, skipping')
                continue

            try:
                subprocess_wrapper(['cp', '-r', source_path, dest_path], timeout=300)
            except subprocess.TimeoutExpired:
                print(f'  WARNING: Timeout copying {pod_dir} (>5 minutes)')
            except subprocess.CalledProcessError as e:
                print(f'  WARNING: Failed to copy {pod_dir}')
            except Exception as e:
                print(f'  WARNING: Error copying {pod_dir}: {str(e)}')


def cmd_outputs(cmd_dict, support_path):
    devnull = open(os.devnull, 'w')
    failed_cmds = ''
    try:
        for file in cmd_dict:
            if file == 'systemctl.txt':
                output = subprocess_wrapper(cmd_dict[file]).split()
                services = set()
                for word in output:
                    if 'service' in word:
                        services.add(word)

                properties = 'Names,ActiveState,UnitFilePreset,MainPID,ActiveEnterTimestamp'
                run_cmd = ['systemctl', 'show', '-p', properties] + list(services)
            elif file == 'netq-app-admin.txt':
                output = subprocess_wrapper(cmd_dict[file].split())
                with open(NETQ_CMD_SCRIPT, 'w') as fd:
                    os.chmod("netq_script.sh", 0o744)
                    fd.write("#!/bin/sh\n")
                    for container in output.splitlines():
                        fd.write("echo %%%%% Logs of {} %%%%%\n".format(container))
                        fd.write("crictl logs {}\n".format(container))
                    run_cmd = './' + NETQ_CMD_SCRIPT
            elif type(cmd_dict[file]) == str:
                # call this in a temp bash script
                with open(NETQ_CMD_SCRIPT, 'w') as fd:
                    os.chmod("netq_script.sh", 0o744)
                    fd.write("#!/bin/sh\n")
                    fd.write(cmd_dict[file] + "\n")
                    run_cmd = './' + NETQ_CMD_SCRIPT
            elif file == 'kafka-connector-status.txt' :
                try:
                    KAFKA_CONNECT_SERVICE_IP=subprocess.check_output(KAFKA_CONNECT_SERVICE_IP_COMMAND,text=True, timeout=CMD_TIMEOUT,stderr=devnull)
                    run_cmd = cmd_dict[file]
                    run_cmd[1] = run_cmd[1].format(service_ip=KAFKA_CONNECT_SERVICE_IP)
                except Exception:
                    pass
            elif file == 'netqd-file-descriptors.txt':
                # Special handling for file descriptor count
                fd_info = get_netqd_fd_count()
                with open(file, 'w') as fp:
                    fp.write(fd_info)
                subprocess_wrapper(['mv', file, support_path + '/Support'])
                continue
            else:
                run_cmd = cmd_dict[file]

            with open(file, 'w') as fp:
                try:
                    subprocess.check_call(run_cmd, stdout=fp, stderr=devnull, timeout=CMD_TIMEOUT)
                    subprocess_wrapper(['mv', file, support_path + '/Support'])
                except Exception as ex:
                    reattempt_status = RC_FAIL
                    if is_opta() and ' '.join(run_cmd) == "netq show agents":
                        with open(file, 'r') as fp1:
                            line = fp1.readline()
                            if "Login Failed" in line or "Access key is not found" in line:
                                print("Access key is not found. Please check the access key entered or generate a fresh access_key,secret_key pair and add it to the CLI configuration")
                                print("Proceeding with opta-support generation without netq show outputs")
                    subprocess_wrapper(['rm', '-f', file])
                    if reattempt_status != RC_SUCCESS:
                        failed_cmds += "Command: %s, Error Output: %s\n" % (' '.join(run_cmd), str(ex))

        if failed_cmds:
            fd_failed_cmds = open(FAILED_COMMANDS_FILE, 'a')
            fd_failed_cmds.write(failed_cmds)
            fd_failed_cmds.close()
            subprocess_wrapper(['mv', FAILED_COMMANDS_FILE, support_path + '/Support'])

            try:
                os.remove(NETQ_CMD_SCRIPT)
            except Exception:
                pass

    except subprocess.CalledProcessError as ex:
        print('ERROR: failed to {}'.format(ex.cmd))
        return RC_FAIL
    else:
        return RC_SUCCESS


def _read_config_from_file(filename):
    """ Read config from file.

    Args:
        filename(str): Name of config file.

    Returns:
        string: Content of config file if file exists else an empty string.
    """
    content = None
    if os.path.exists(filename):
        with open(filename, "r") as fd:
            content = fd.readline()
    return content


def _get_nvl_worker_ips_from_kubectl():
    """Fallback: get NVL worker IPs from kubectl when worker_ip file doesn't exist."""
    try:
        master_ip_raw = _read_config_from_file(MASTER_IP_FILE)
        master_ip = master_ip_raw.strip() if master_ip_raw else None
        result = subprocess.run(
            ["kubectl", "get", "nodes", "-o",
             "jsonpath={.items[*].status.addresses[?(@.type==\"InternalIP\")].address}"],
            capture_output=True, text=True, check=True)
        all_ips = result.stdout.strip().split()
        if master_ip:
            return [ip for ip in all_ips if ip and ip != master_ip]
        return [ip for ip in all_ips if ip]
    except Exception as e:
        print(f"WARNING: Failed to get NVL worker IPs from kubectl: {e}")
        return []


def _read_worker_ip_config_from_file(filename):
    """
    Read multi line config from file.
    Args:
        filename(str): Name of config file.

    Returns:
        string: Content of config file if file exists else an empty string.
    """
    worker_ips = None
    if os.path.exists(filename):
        with open(filename, "r") as fd:
            worker_ips = [line.strip() for line in fd]
    return worker_ips


def _is_standalone(is_worker):
    """
    Determine if this is a standalone installation.
    
    Returns:
        bool: True if standalone, False if cluster (master or worker).
    """
    if is_worker:
        return False

    master_ip = _read_config_from_file(MASTER_IP_FILE)
    
    # Get local IPs
    result = subprocess.run(['hostname', '-I'], capture_output=True, text=True)
    local_ips = set(ip.strip() for ip in result.stdout.strip().split() if ip.strip())
    
    # If master_ip is not in local_ips, this is a worker node (not standalone)
    if master_ip and master_ip not in local_ips:
        return False
    
    # Check if worker_ip file has IPs (indicates cluster master)
    worker_ip_list = _read_worker_ip_config_from_file(WORKER_IP_FILE)
    is_cluster = bool(worker_ip_list and [ip for ip in worker_ip_list if ip.strip()])
    
    # Standalone = master_ip matches local AND no workers configured
    return not is_cluster


def _run_on_worker(worker_ip, command):
    """ Method to run a command on worker node.

    Args:
        worker_ip: Ip of worker node.
        command: Command to run.

    Returns: None

    Raises: None
    """
    ssh_cmd = 'ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 -qi {}/master ' \
                          'netq-admin@{} sudo {}'.format(SSH_DIR, worker_ip, command)

    if is_bcm():
        ssh_cmd = 'ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 ' \
                          'root@{} sudo {}'.format(worker_ip, command)
    return subprocess.check_output(ssh_cmd, shell=True, stderr=subprocess.DEVNULL)


def _copy_from_worker_to_master(worker_ip, remote_file_path, local_path):
    """ Method to run a command on worker node.

    Args:
        worker_ip: Ip of worker node.
        command: Command to run.

    Returns: None

    Raises: None
    """
    ssh_cmd = 'scp -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 -qi {}/master ' \
                          'netq-admin@{}:{} {}'.format(SSH_DIR, worker_ip, remote_file_path , local_path)

    if is_bcm():
        ssh_cmd = 'scp -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5 ' \
                          'root@{}:{} {}'.format(worker_ip, remote_file_path , local_path)
    return subprocess.check_output(ssh_cmd, shell=True, stderr=subprocess.DEVNULL)


def combine_tarballs(WORKER_SUPPORT_FILE_LIST):
    if is_opta():
        COMBINED_SUPPORT_NAME = OPTAPREFIX + '_' + BASENAME_COMBINED
        MASTER_SUPPORT_NAME = OPTAPREFIX + '_' + BASENAME
    else:
        COMBINED_SUPPORT_NAME = NETQPREFIX + '_' + BASENAME_COMBINED
        MASTER_SUPPORT_NAME = NETQPREFIX + '_' + BASENAME
    COMBINED_SUPPORT_FILE = '%s%s%s' % (SUPPORT_PATH, COMBINED_SUPPORT_NAME, EXTENSION)
    MASTER_SUPPORT_FILE = '%s%s' % (MASTER_SUPPORT_NAME, EXTENSION)
    COMBINED_SUPPORT_FILE_LIST = WORKER_SUPPORT_FILE_LIST + [MASTER_SUPPORT_FILE]
    if not COMBINED_SUPPORT_FILE_LIST:
        print("No tar files found in the directory.")
        return
    try:
        subprocess_wrapper(['/bin/tar', '--directory=/var/support', TAR_OPTIONS, COMBINED_SUPPORT_FILE] + COMBINED_SUPPORT_FILE_LIST)
        print('Please send {} to Nvidia support.'.format(COMBINED_SUPPORT_FILE))
    except subprocess.CalledProcessError as ex:
        print('ERROR: could not create combined support file ({})'.format(ex.cmd))
        pass


def is_node_up(host):
    try:
        response = subprocess.run(
            ["ping", "-c", "1", host],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            timeout=5
        )
        return response.returncode == 0
    except subprocess.TimeoutExpired:
        return False


def create_opta_support_on_worker(worker_ip_list, include_pod_logs=False):
    command = 'is_worker=True opta-support'
    if include_pod_logs:
        command += ' --include-pod-logs'
    failed_workers = []
    with futures.ThreadPoolExecutor() as executor:
        future_to_ip = {executor.submit(_run_on_worker, worker_ip, command): worker_ip for worker_ip in worker_ip_list}
        for future in futures.as_completed(future_to_ip):
            ip = future_to_ip[future]
            msg = f'WARNING: opta-support failed on worker {ip}'
            try:
                output = future.result().decode('utf8').strip()
                if "ERROR" in output:
                    print(f'{msg}: {output}')
                    failed_workers.append(ip)
            except Exception as e:
                print(f'{msg}: {str(e)}')
                failed_workers.append(ip)
    return failed_workers


def create_unified_support_structure(is_worker, is_eth=True, is_nvl=False, is_standalone=False):
    """Create unified support directory structure.

    Layout:
      - cluster master: kafka_modules/, observability_data/, master_logs/{eth_logs,nvl_logs}/
      - standalone master: kafka_modules/, observability_data/, eth_logs/, nvl_logs/ (flat)
      - worker: eth_logs/, nvl_logs/ at the archive root (master extracts it as <host>_logs/)
    """
    # Use different basename for standalone vs cluster
    if is_standalone:
        base_name = BASENAME
    else:
        base_name = BASENAME_COMBINED

    unified_support_path = f'/tmp/{base_name}'
    observability_path = f'{unified_support_path}/observability_data'
    kafka_modules_path = f'{unified_support_path}/kafka_modules'

    # eth_logs/nvl_logs live under master_logs/ only on a cluster master, otherwise at the root
    if not is_worker and not is_standalone:
        logs_parent = f'{unified_support_path}/master_logs'
    else:
        logs_parent = unified_support_path
    eth_logs_path = f'{logs_parent}/eth_logs'
    nvl_logs_path = f'{logs_parent}/nvl_logs'

    subprocess_wrapper(['mkdir', '-p', unified_support_path])
    if not is_worker:
        subprocess_wrapper(['mkdir', '-p', kafka_modules_path])
    if is_eth:
        subprocess_wrapper(['mkdir', '-p', eth_logs_path])
    if is_nvl:
        subprocess_wrapper(['mkdir', '-p', nvl_logs_path])
        if not is_worker:
            subprocess_wrapper(['mkdir', '-p', observability_path])

    return unified_support_path, eth_logs_path, nvl_logs_path, observability_path, kafka_modules_path


def _collect_logs_to_directory(args, output_dir, is_cluster, namespace):
    """Collect pod logs locally (on this node) for the given namespace."""
    opta_support_to_directory(args, output_dir, is_cluster, pod_log_namespaces=[namespace])
    return RC_SUCCESS


def _flatten_worker_archive(local_archive, output_dir, worker_host):
    """Extract a worker support archive into <worker_host>_logs/ under output_dir.

    Uses tar --strip-components=1 to drop the archive's top-level support_combined_*/
    wrapper, so eth_logs/ and nvl_logs/ land directly under <worker_host>_logs/.
    Returns True on success so the caller can delete the original archive.
    """
    target_path = os.path.join(output_dir, f'{worker_host}_logs')
    try:
        subprocess_wrapper(['mkdir', '-p', target_path])
        subprocess_wrapper(['/bin/tar', '-xf', local_archive,
                            '-C', target_path, '--strip-components=1'])
    except subprocess.CalledProcessError as ex:
        print(f'WARNING: Failed to extract worker archive {local_archive}: {ex}')
        return False
    return True


def _collect_worker_logs(worker_ip_list, output_dir, include_pod_logs):
    """Trigger opta-support on worker nodes once and copy their archives to output_dir."""
    active_worker_ip_list = []
    for worker_ip in worker_ip_list:
        if not is_node_up(worker_ip):
            print("ERROR : node ip {} is unreachable. Skipping opta-support for this node".format(worker_ip))
            continue
        active_worker_ip_list.append(worker_ip)
    failed_workers = []
    try:
        failed_workers = create_opta_support_on_worker(active_worker_ip_list, include_pod_logs)
    except Exception as e:
        print("Failed to collect opta-support from worker nodes. Please check if worker nodes are initialized and reachable and try again.")
    # Exclude workers where create opta-support failed
    active_worker_ip_list = [ip for ip in active_worker_ip_list if ip not in failed_workers]
    for worker_ip in active_worker_ip_list:
        msg = f'WARNING: Failed to get latest opta-support file on worker {worker_ip}'
        command = '"ls -t /var/support/*.txz | head -n 1"'
        latest_file = ""
        try:
            latest_file = _run_on_worker(worker_ip, command).decode('utf8').strip()
            worker_host = _run_on_worker(worker_ip, 'hostname').decode('utf8').strip()
            if "ERROR" in latest_file or not worker_host:
                print(f'{msg}: {latest_file}')
                continue
        except subprocess.CalledProcessError as e:
            print(f'{msg}: {str(e)}')
            continue

        if not latest_file:
            continue

        msg = f'WARNING: Failed to copy latest opta-support file on {worker_ip} to {output_dir}'
        copy_ok = False
        try:
            output = _copy_from_worker_to_master(worker_ip, latest_file, output_dir).decode('utf8').strip()
            if "ERROR" in output:
                print(f'{msg}: {output}')
            else:
                copy_ok = True
        except subprocess.CalledProcessError as e:
            print(f'{msg}: {str(e)}')

        if not copy_ok:
            continue

        local_archive = os.path.join(output_dir, os.path.basename(latest_file))
        if _flatten_worker_archive(local_archive, output_dir, worker_host):
            try:
                os.remove(local_archive)
            except OSError as ex:
                print(f'WARNING: Failed to remove worker archive {local_archive}: {ex}')


def opta_support_to_directory(args, output_dir, is_cluster, pod_log_namespaces=None):
    prog_name = os.path.basename(sys.argv[0])
    copy_db_logs = args.include_db_logs

    if os.getuid():
        print('Must run {} as root'.format(prog_name))
        sys.exit(1)

    if is_opta():
        print('Generating opta-support archive. Process takes few minutes to complete...')
    file_dict, cmd_dict = platform()

    TMP_SUPPORT_PATH = output_dir

    subprocess_wrapper(['mkdir', '-p', TMP_SUPPORT_PATH])
    subprocess_wrapper(['mkdir', '-p', TMP_SUPPORT_PATH + '/Logs'])
    subprocess_wrapper(['mkdir', '-p', TMP_SUPPORT_PATH + '/Support'])

    if is_cumulus():
        CUMULUS_SUPPORT_NAME = '/cl-support'
        CL_SUPPORT_PATH = TMP_SUPPORT_PATH + (CUMULUS_SUPPORT_NAME)
        subprocess_wrapper(['mkdir', CL_SUPPORT_PATH])
        print('Collecting cl-support...')
        try:
            subprocess_wrapper(['cl-support', '-s', '-S', CL_SUPPORT_PATH, '-p', \
                                'netq-cl', '-r', '\"triggered from netq-support\"', '-T', '300', '-j'])
            print('Collecting netq-support...')
        except subprocess.CalledProcessError as ex:
            print('cl-support collection did not succeed. collecting netq-support alone')
            pass

        nvue_client = None
        try:
            # Use NVUE API client to get platform information
            from netq_agent.cmd.nvue_api_client import create_nvue_client
            nvue_client = create_nvue_client()
            json_output = nvue_client.get_platform_hardware()

            if json_output and json_output.get("asic-model") not in ["Spectrum", "n/a"]:
                NETQ_SPICE_SUPPORT = 'python /sbin/netq-spice-support'
                subprocess_wrapper(NETQ_SPICE_SUPPORT.split())

        except Exception as e:
            print("Error getting platform information via NVUE API:", e)
            pass
        finally:
            # Clean up NVUE client connection
            if nvue_client:
                nvue_client.close()

    cmd_outputs(cmd_dict, TMP_SUPPORT_PATH)
    sleep(3)
    copy_files(file_dict, TMP_SUPPORT_PATH)
    if 'copy_db_logs' in locals() and copy_db_logs:
        copy_pods_files(TMP_SUPPORT_PATH)

    # Collect complete /var/log/pods for the specified namespaces if include-pod-logs flag is provided
    if is_opta() and args.include_pod_logs:
        namespaces = pod_log_namespaces if pod_log_namespaces is not None else ['netq-eth']
        collect_var_log_pods(TMP_SUPPORT_PATH, namespaces=namespaces)


def collect_nvl_observability_data_to_directory(output_dir, img_build_version):
    try:
        # Change to the output directory before running the script
        original_cwd = os.getcwd()
        os.chdir(output_dir)

        if not img_build_version:
            raise ValueError("ERROR: Build version is not recognized, cannot collect NVL observability data")

        # Run the observability data extraction script
        result = subprocess.run(["bash", "/opt/netq-admin/nvl/scripts/extract-observability-data.sh", str(is_bcm()).lower(), img_build_version])

        # Change back to original directory
        os.chdir(original_cwd)

        if result.returncode != 0:
            print(f"WARNING: extract-observability-data.sh failed with return code {result.returncode}")
            return RC_FAIL

        return RC_SUCCESS

    except Exception as e:
        print(f"ERROR: Failed to collect observability data: {e}")
        os.chdir(original_cwd)
        return RC_FAIL


def collect_kafka_modules_to_directory(kafka_modules_dir, include_kafka_modules):
    try:
        log_file = os.path.join(kafka_modules_dir, "netq-kafka-modules-output.log")

        # Open the log file for writing
        with open(log_file, "w") as fout:
        # Run the kafka support collection script
            if include_kafka_modules != None:
                result = subprocess.run([
                "/usr/sbin/netq-kafka-support-collect.py",
                "--include", include_kafka_modules,
                "--output-directory", kafka_modules_dir
                ],
                stdout=fout,
                stderr=subprocess.STDOUT,
                text=True)
            else:
                result = subprocess.run([
                "/usr/sbin/netq-kafka-support-collect.py",
                "--output-directory", kafka_modules_dir
                ],
                stdout=fout,
                stderr=subprocess.STDOUT,
                text=True)

        if result.returncode != 0:
            # Kafka support collection is optional - don't print warning to console            
            return RC_FAIL

        return RC_SUCCESS

    except Exception as e:
        print(f"ERROR: Failed to collect kafka modules: {e}")
        return RC_FAIL


def get_installation_mode():
    try:
        netq_status = subprocess.run(["netq", "show", "status", "json"], capture_output=True, text=True, check=True)
        netq_status_json = json.loads(netq_status.stdout)
        installation_mode = netq_status_json.get("installation_mode", "").lower()
        img_build_version = netq_status_json.get("installer_version", "")

        modes = {
            "combined": (True, True),
            "nvlink": (False, True),
            "ethernet": (True, False),
        }

        # in case the installation mode isn't available, run the default OPTA-support - Ethernet.
        is_eth_installed, is_nvl_installed = modes.get(installation_mode, (True, False))
        return is_eth_installed, is_nvl_installed, img_build_version

    except Exception as e:
        print(f"Failed to get installation status, proceeding with default values.")
        # Return default values: Ethernet mode enabled, NVL disabled, empty version
        # This ensures the function always returns a tuple for unpacking
        return True, False, ""


def check_collection_results(is_eth, eth_result, is_nvl, nvl_logs_result, nvl_observability_result, kafka_result=None):
    if is_eth and eth_result != RC_SUCCESS:
        print("ERROR: Ethernet log collection failed")
    if is_nvl and nvl_logs_result != RC_SUCCESS:
        print("ERROR: NVL pod log collection failed")
    if is_nvl and nvl_observability_result != RC_SUCCESS:
        print("ERROR: NVL observability data collection failed")
    #if kafka_result is not None and kafka_result != RC_SUCCESS:
    #    print("ERROR: Kafka modules collection failed")


def cleanup_temp_dir(path):
    try:
        if os.path.exists(path):
            subprocess_wrapper(['rm', '-rf', path])
    except Exception as e:
        print(f"WARNING: Failed to clean up temporary directory {path}: {e}")


def create_unified_archive(unified_path):
    unified_basename = os.path.basename(unified_path)
    unified_tar_file = f'{SUPPORT_PATH}{unified_basename}{EXTENSION}'

    try:
        if not os.path.isdir(SUPPORT_PATH):
            subprocess_wrapper(['mkdir', SUPPORT_PATH])
        if 'J' in TAR_OPTIONS: # xz compression with multi-threading
            subprocess_wrapper(['/bin/tar', '--directory=/tmp',
            '-I', f'xz -T{XZ_THREADS}', '-cf', unified_tar_file, unified_basename])
        else: # gzip compression
            subprocess_wrapper(['/bin/tar', '--directory=/tmp',
            TAR_OPTIONS, unified_tar_file, unified_basename])
        print(f'Please send {unified_tar_file} to Nvidia support.')
    except subprocess.CalledProcessError as ex:
        print(f'ERROR: could not create unified support file ({str(ex)})')
        print('returncode:', ex.returncode)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--include-db-logs', action="store_true", help="Include database logs")
    parser.add_argument('--include-kafka-modules', required=False, help="Include kafka modules, e.g. apps:node, apps:*")
    parser.add_argument('--include-pod-logs', action="store_true",
                        help="Include complete pod logs for netq-eth (ETH mode) and netq-nvl (NVL mode) namespaces")
    args = parser.parse_args()

    is_worker = os.getenv('is_worker')

    if is_cumulus() and not is_opta():
        prog_name = os.path.basename(sys.argv[0])
        copy_db_logs = args.include_db_logs

        if os.getuid():
            print('Must run {} as root'.format(prog_name))
            sys.exit(1)

        file_dict, cmd_dict = platform()
        SUPPORT_NAME = NETQPREFIX + '_support_' + NETQ_BASENAME
        SUPPORT_FILE = '%s%s%s' % (SUPPORT_PATH, SUPPORT_NAME, EXTENSION)
        TMP_SUPPORT_PATH = '/tmp/' + SUPPORT_NAME

        subprocess_wrapper(['mkdir', '-p', TMP_SUPPORT_PATH])
        subprocess_wrapper(['mkdir', TMP_SUPPORT_PATH + '/Logs'])
        subprocess_wrapper(['mkdir', TMP_SUPPORT_PATH + '/Support'])

        # Cumulus-specific collection
        CUMULUS_SUPPORT_NAME = '/cl-support'
        CL_SUPPORT_PATH = TMP_SUPPORT_PATH + (CUMULUS_SUPPORT_NAME)
        subprocess_wrapper(['mkdir', CL_SUPPORT_PATH])
        print('Collecting cl-support...')
        try:
            subprocess_wrapper(['cl-support', '-s', '-S', CL_SUPPORT_PATH, '-p', \
                                'netq-cl', '-r', '\"triggered from netq-support\"', '-T', '300'])
            print('Collecting netq-support...')
        except subprocess.CalledProcessError as ex:
            print('cl-support collection did not succeed. collecting netq-support alone')
            pass

        nvue_client = None
        try:
            # Use NVUE API client to get platform information
            from netq_agent.cmd.nvue_api_client import create_nvue_client
            nvue_client = create_nvue_client()
            json_output = nvue_client.get_platform_hardware()

            if json_output and json_output.get("asic-model") not in ["Spectrum", "n/a"]:
                NETQ_SPICE_SUPPORT = 'python /sbin/netq-spice-support'
                subprocess_wrapper(NETQ_SPICE_SUPPORT.split())

        except Exception as e:
            print("Error getting platform information via NVUE API:", e)
            pass
        finally:
            # Clean up NVUE client connection
            if nvue_client:
                nvue_client.close()

        cmd_outputs(cmd_dict, TMP_SUPPORT_PATH)
        sleep(3)
        copy_files(file_dict, TMP_SUPPORT_PATH)
        if 'copy_db_logs' in locals() and copy_db_logs:
            copy_pods_files(TMP_SUPPORT_PATH)

        try:
            if not os.path.isdir(SUPPORT_PATH):
                subprocess_wrapper(['mkdir', SUPPORT_PATH])
            subprocess_wrapper(['/bin/tar', '--directory=/tmp', TAR_OPTIONS, SUPPORT_FILE, SUPPORT_NAME])
            print('Please send {} to Nvidia support.'.format(SUPPORT_FILE))
        except subprocess.CalledProcessError as ex:
            print('ERROR: could not create support file ({})'.format(ex.cmd))

        # Clean up temporary directory
        try:
            subprocess_wrapper(['rm', '-rf', TMP_SUPPORT_PATH])
        except Exception as e:
            print(f"WARNING: Failed to clean up temporary directory {TMP_SUPPORT_PATH}: {e}")
        return

    # For OPTA systems, use unified support structure
    eth_success = RC_SUCCESS
    nvl_logs_success = RC_SUCCESS
    observability_success = RC_SUCCESS
    kafka_success = RC_SUCCESS

    is_standalone = _is_standalone(is_worker)

    unified_path = None
    try:
        is_eth_installed, is_nvl_installed, img_build_version = get_installation_mode()

        unified_path, eth_logs_dir, nvl_logs_dir, observability_dir, kafka_modules_dir = create_unified_support_structure(
            is_worker, is_eth=is_eth_installed, is_nvl=is_nvl_installed, is_standalone=is_standalone)

        worker_ip_list = _read_worker_ip_config_from_file(WORKER_IP_FILE) if not is_worker else []
        # Fallback for NVL clusters where worker_ip file may not exist yet
        if not worker_ip_list and is_nvl_installed and not is_worker:
            worker_ip_list = _get_nvl_worker_ips_from_kubectl()
        is_cluster = bool(worker_ip_list and [ip for ip in worker_ip_list if ip.strip()])

        if is_eth_installed:
            eth_success = _collect_logs_to_directory(args, eth_logs_dir, is_cluster, namespace='netq-eth')

        if is_nvl_installed:
            nvl_logs_success = _collect_logs_to_directory(args, nvl_logs_dir, is_cluster, namespace='netq-nvl')

        # Trigger worker collection once — each worker runs opta-support and produces a single archive
        # containing all installed-mode logs. The archive is extracted in place under unified_path
        # as <worker_host>_logs/, so it always lands at the unified root regardless of installed modes.
        if is_cluster:
            _collect_worker_logs(worker_ip_list, unified_path, args.include_pod_logs)

        if is_nvl_installed and not is_worker:
            observability_success = collect_nvl_observability_data_to_directory(observability_dir, img_build_version)

        # Collect kafka modules if not worker node
        if not is_worker:
            kafka_success = collect_kafka_modules_to_directory(kafka_modules_dir, args.include_kafka_modules)

        # Indicate if either collection failed
        check_collection_results(is_eth_installed, eth_success, is_nvl_installed, nvl_logs_success, observability_success, kafka_success)

        # Create unified tar file
        if eth_success == RC_SUCCESS or nvl_logs_success == RC_SUCCESS or observability_success == RC_SUCCESS or kafka_success == RC_SUCCESS:
            create_unified_archive(unified_path)
        else:
            print("ERROR: All collection tasks failed")

    except Exception as e:
        print(f"ERROR: logs collection failed: {e}")

    finally:
        if unified_path:
            cleanup_temp_dir(unified_path)


if __name__ == '__main__':
    main()
