From f64d190fcacede2469becf5204a490402415746f Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 12:28:09 +0200 Subject: [PATCH 01/12] check CUPS --- fedcloud_vm_monitoring/site_monitor.py | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index 4c6b510..f89c0ff 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -12,6 +12,7 @@ from fedcloudclient.sites import find_endpoint_and_project_id from ldap3.core.exceptions import LDAPException from paramiko import SSHException +import subprocess class SiteMonitorException(Exception): @@ -238,6 +239,42 @@ def get_sshd_version(self, ip_addresses): else: return "No public IP available to check SSH version." + def _run_shell_command(self, command): + completed = subprocess.run( + command, + shell=True, + capture_output=True, + text=True, + ) + returncode = completed.returncode + stdout = completed.stdout + stderr = completed.stderr + return returncode, stdout, stderr + + def check_open_port(self, ip, port, protocol): + if protocol == "tcp": + command = f"ncat --nodns -z {ip} {port}" + elif protocol == "udp": + command = f"ncat --udp --nodns --idle-timeout 3s {ip} {port}" + else: + raise SiteMonitorException(f"Protocol {protocol} not supported!") + returncode, stdout, stderr = self._run_shell_command(command) + return returncode, stdout, stderr + + def check_CUPS(self, ip_addresses): + public_ip = self.get_public_ip(ip_addresses) + if len(public_ip) > 0: + returncode_tcp, stdout_tcp, stderr_tcp = self.check_open_port(public_ip, 631, "tcp") + returncode_upd, stdout_upd, stderr_upd = self.check_open_port(public_ip, 631, "udp") + if returncode_tcp == 0 or returncode_upd == 0: + return "WARNING: CUPS port is open" + elif returncode_tcp == 1 and returncode_upd == 1: + return "CUPS port is not open" + else: + return "Error checking CUPS port: " + returncode + stdout + stderr + else: + return "No public IP available to check CUPs version" + def process_vm(self, vm): vm_info = self.get_vm(vm) flv = self.get_flavor(vm["Flavor"]) @@ -245,6 +282,7 @@ def process_vm(self, vm): for net, addrs in vm["Networks"].items(): vm_ips.extend(addrs) sshd_version = self.get_sshd_version(vm_ips) + CUPS_check = self.check_CUPS(vm_ips) created = parse(vm_info["created_at"]) elapsed = self.now - created output = [ @@ -253,6 +291,7 @@ def process_vm(self, vm): ("status", click.style(vm["Status"], fg=self.color_maps[vm["Status"]])), ("ip address", " ".join(vm_ips)), ("SSH version", sshd_version), + ("CUPS", CUPS_check), ] if flv: output.append( From 8c30ba012b66c1d718adae2e7fda7acc21b6c861 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 13:42:47 +0200 Subject: [PATCH 02/12] check whether ncat is installed --- fedcloud_vm_monitoring/site_monitor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index f89c0ff..60a1da8 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -262,6 +262,9 @@ def check_open_port(self, ip, port, protocol): return returncode, stdout, stderr def check_CUPS(self, ip_addresses): + returncode_ncat, stdout_ncat, stderr_ncat = self._run_shell_command("which ncat") + if returncode_ncat != 0: + return "ncat ( https://nmap.org/ncat ) is not installed" public_ip = self.get_public_ip(ip_addresses) if len(public_ip) > 0: returncode_tcp, stdout_tcp, stderr_tcp = self.check_open_port(public_ip, 631, "tcp") @@ -269,7 +272,7 @@ def check_CUPS(self, ip_addresses): if returncode_tcp == 0 or returncode_upd == 0: return "WARNING: CUPS port is open" elif returncode_tcp == 1 and returncode_upd == 1: - return "CUPS port is not open" + return "CUPS port is closed" else: return "Error checking CUPS port: " + returncode + stdout + stderr else: From 4ba99763b644d79a7cd319e55a00f4c5bdc5231e Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:15:29 +0200 Subject: [PATCH 03/12] make CUPS and SSH checks optional and False by default --- README.md | 3 +++ fedcloud_vm_monitoring/cli.py | 16 +++++++++++++++- fedcloud_vm_monitoring/site_monitor.py | 12 +++++++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1b71aa2..b2b3afe 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,9 @@ You can tune the behavior with the following parameters: triggering deletion (default 90 days). - `--show-quotas BOOLEAN`: whether to show quotas for the VO or not (default: `True`) +- `--check-ssh BOOLEAN`: Check SSH version on target VMs (default: `False`) +- `--check-cups BOOLEAN`: Check whether TCP/UDP port 631 is accessible + (default: `False`) If you have access to [Check-in LDAP](https://docs.egi.eu/users/aai/check-in/vos/#ldap) for VO diff --git a/fedcloud_vm_monitoring/cli.py b/fedcloud_vm_monitoring/cli.py index ace5b0d..d62e483 100644 --- a/fedcloud_vm_monitoring/cli.py +++ b/fedcloud_vm_monitoring/cli.py @@ -35,6 +35,18 @@ help="Show quotas for VO", show_default=True, ) +@click.option( + "--check-ssh", + default=False, + help="Check SSH version on target VMs", + show_default=True, +) +@click.option( + "--check-cups", + default=False, + help="Check whether TCP/UDP port 631 is accessible", + show_default=True, +) @click.option( "--ldap-server", default="ldaps://ldap.aai.egi.eu:636", @@ -62,6 +74,8 @@ def main( max_days, delete, show_quotas, + check_ssh, + check_cups, ldap_server, ldap_base_dn, ldap_user, @@ -85,7 +99,7 @@ def main( sites = [site] if site else set(appdb_sites + fedcloudclient_sites) for s in sites: click.secho(f"[.] Checking VO {vo} at {s}", fg="blue", bold=True) - site_monitor = SiteMonitor(s, vo, access_token, max_days, ldap_config) + site_monitor = SiteMonitor(s, vo, access_token, max_days, check_ssh, check_cups, ldap_config) try: site_monitor.vm_monitor(delete) except SiteMonitorException as e: diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index 60a1da8..672d129 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -28,11 +28,13 @@ class SiteMonitor: min_secgroup_instance_ratio = 3 min_ip_instance_ratio = 1 - def __init__(self, site, vo, token, max_days, ldap_config={}): + def __init__(self, site, vo, token, max_days, check_ssh, check_cups, ldap_config={}): self.site = site self.vo = vo self.token = token self.max_days = max_days + self.check_ssh = check_ssh + self.check_cups = check_cups self.ldap_config = ldap_config self.flavors = {} self.users = defaultdict(lambda: {}) @@ -293,9 +295,13 @@ def process_vm(self, vm): ("instance id", vm["ID"]), ("status", click.style(vm["Status"], fg=self.color_maps[vm["Status"]])), ("ip address", " ".join(vm_ips)), - ("SSH version", sshd_version), - ("CUPS", CUPS_check), ] + if self.check_ssh: + sshd_version = self.get_sshd_version(vm_ips) + output.append(("SSH version", sshd_version)) + if self.check_cups: + CUPS_check = self.check_CUPS(vm_ips) + output.append(("CUPS", CUPS_check)) if flv: output.append( ( From 9bcfb0873dd40ede58439024b09a288939d62a01 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:15:44 +0200 Subject: [PATCH 04/12] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bec73cc..bcba9f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fedcloud-vm-monitoring" -version = "0.2.0" +version = "0.2.1" description = "Monitoring fedcloud VMs and sites" authors = ["Giuseppe La Rocca ", "Enol Fernandez "] From 24f8b948ed943fdb87486860a709dbce32226442 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:24:56 +0200 Subject: [PATCH 05/12] linting --- fedcloud_vm_monitoring/cli.py | 4 ++- fedcloud_vm_monitoring/site_monitor.py | 38 ++++++++++++++++---------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/fedcloud_vm_monitoring/cli.py b/fedcloud_vm_monitoring/cli.py index d62e483..ba7c6c2 100644 --- a/fedcloud_vm_monitoring/cli.py +++ b/fedcloud_vm_monitoring/cli.py @@ -99,7 +99,9 @@ def main( sites = [site] if site else set(appdb_sites + fedcloudclient_sites) for s in sites: click.secho(f"[.] Checking VO {vo} at {s}", fg="blue", bold=True) - site_monitor = SiteMonitor(s, vo, access_token, max_days, check_ssh, check_cups, ldap_config) + site_monitor = SiteMonitor( + s, vo, access_token, max_days, check_ssh, check_cups, ldap_config) + ) try: site_monitor.vm_monitor(delete) except SiteMonitorException as e: diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index 672d129..64fc5d3 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -1,6 +1,7 @@ """Monitor VM instances running in the provider""" import ipaddress +import subprocess from collections import defaultdict from datetime import datetime, timezone @@ -12,7 +13,6 @@ from fedcloudclient.sites import find_endpoint_and_project_id from ldap3.core.exceptions import LDAPException from paramiko import SSHException -import subprocess class SiteMonitorException(Exception): @@ -28,7 +28,9 @@ class SiteMonitor: min_secgroup_instance_ratio = 3 min_ip_instance_ratio = 1 - def __init__(self, site, vo, token, max_days, check_ssh, check_cups, ldap_config={}): + def __init__( + self, site, vo, token, max_days, check_ssh, check_cups, ldap_config={} + ): self.site = site self.vo = vo self.token = token @@ -242,16 +244,16 @@ def get_sshd_version(self, ip_addresses): return "No public IP available to check SSH version." def _run_shell_command(self, command): - completed = subprocess.run( - command, - shell=True, - capture_output=True, - text=True, - ) - returncode = completed.returncode - stdout = completed.stdout - stderr = completed.stderr - return returncode, stdout, stderr + completed = subprocess.run( + command, + shell=True, + capture_output=True, + text=True, + ) + returncode = completed.returncode + stdout = completed.stdout + stderr = completed.stderr + return returncode, stdout, stderr def check_open_port(self, ip, port, protocol): if protocol == "tcp": @@ -264,13 +266,19 @@ def check_open_port(self, ip, port, protocol): return returncode, stdout, stderr def check_CUPS(self, ip_addresses): - returncode_ncat, stdout_ncat, stderr_ncat = self._run_shell_command("which ncat") + returncode_ncat, stdout_ncat, stderr_ncat = self._run_shell_command( + "which ncat" + ) if returncode_ncat != 0: return "ncat ( https://nmap.org/ncat ) is not installed" public_ip = self.get_public_ip(ip_addresses) if len(public_ip) > 0: - returncode_tcp, stdout_tcp, stderr_tcp = self.check_open_port(public_ip, 631, "tcp") - returncode_upd, stdout_upd, stderr_upd = self.check_open_port(public_ip, 631, "udp") + returncode_tcp, stdout_tcp, stderr_tcp = self.check_open_port( + public_ip, 631, "tcp" + ) + returncode_upd, stdout_upd, stderr_upd = self.check_open_port( + public_ip, 631, "udp" + ) if returncode_tcp == 0 or returncode_upd == 0: return "WARNING: CUPS port is open" elif returncode_tcp == 1 and returncode_upd == 1: From f19f5dcc8275788a558a7325c6e93aeb6ce57334 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:29:42 +0200 Subject: [PATCH 06/12] linting --- fedcloud_vm_monitoring/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedcloud_vm_monitoring/cli.py b/fedcloud_vm_monitoring/cli.py index ba7c6c2..a967596 100644 --- a/fedcloud_vm_monitoring/cli.py +++ b/fedcloud_vm_monitoring/cli.py @@ -100,7 +100,7 @@ def main( for s in sites: click.secho(f"[.] Checking VO {vo} at {s}", fg="blue", bold=True) site_monitor = SiteMonitor( - s, vo, access_token, max_days, check_ssh, check_cups, ldap_config) + s, vo, access_token, max_days, check_ssh, check_cups, ldap_config ) try: site_monitor.vm_monitor(delete) From 4e7faabf65e2c18992565ee14e5c3d5288936074 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:37:20 +0200 Subject: [PATCH 07/12] linting --- fedcloud_vm_monitoring/site_monitor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index 64fc5d3..c9382d9 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -284,7 +284,9 @@ def check_CUPS(self, ip_addresses): elif returncode_tcp == 1 and returncode_upd == 1: return "CUPS port is closed" else: - return "Error checking CUPS port: " + returncode + stdout + stderr + return "Error checking CUPS port: " + + "TCP return code: {returncode_tcp}, stdout: {stdout_tcp}, stderr: {stderr_tcp}" + + "UDP return code: {returncode_upd}, stdout: {stdout_upd}, stderr: {stderr_upd}" else: return "No public IP available to check CUPs version" From cc18eb5bd7d6c0edddc1827fb41820097ecefab3 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:39:14 +0200 Subject: [PATCH 08/12] linting --- fedcloud_vm_monitoring/site_monitor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index c9382d9..09bc212 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -284,9 +284,10 @@ def check_CUPS(self, ip_addresses): elif returncode_tcp == 1 and returncode_upd == 1: return "CUPS port is closed" else: - return "Error checking CUPS port: " + - "TCP return code: {returncode_tcp}, stdout: {stdout_tcp}, stderr: {stderr_tcp}" + + return ("Error checking CUPS port: " + "TCP return code: {returncode_tcp}, stdout: {stdout_tcp}, stderr: {stderr_tcp}" "UDP return code: {returncode_upd}, stdout: {stdout_upd}, stderr: {stderr_upd}" + ) else: return "No public IP available to check CUPs version" From c5d67843f38b684d804e0b4bf7bccc43d7157cae Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:42:32 +0200 Subject: [PATCH 09/12] linting --- fedcloud_vm_monitoring/site_monitor.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index 09bc212..b6c5020 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -284,10 +284,11 @@ def check_CUPS(self, ip_addresses): elif returncode_tcp == 1 and returncode_upd == 1: return "CUPS port is closed" else: - return ("Error checking CUPS port: " - "TCP return code: {returncode_tcp}, stdout: {stdout_tcp}, stderr: {stderr_tcp}" - "UDP return code: {returncode_upd}, stdout: {stdout_upd}, stderr: {stderr_upd}" - ) + return ( + "Error checking CUPS port: " + "TCP return code: {returncode_tcp}, stdout: {stdout_tcp}, stderr: {stderr_tcp}" + "UDP return code: {returncode_upd}, stdout: {stdout_upd}, stderr: {stderr_upd}" + ) else: return "No public IP available to check CUPs version" From a450601024fc5f75e0e8bacff7793952299150ae Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Fri, 11 Oct 2024 14:44:51 +0200 Subject: [PATCH 10/12] punctuation --- fedcloud_vm_monitoring/site_monitor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index b6c5020..44040d0 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -285,9 +285,9 @@ def check_CUPS(self, ip_addresses): return "CUPS port is closed" else: return ( - "Error checking CUPS port: " - "TCP return code: {returncode_tcp}, stdout: {stdout_tcp}, stderr: {stderr_tcp}" - "UDP return code: {returncode_upd}, stdout: {stdout_upd}, stderr: {stderr_upd}" + "Error checking CUPS port. " + "TCP return code: {returncode_tcp}, stdout: {stdout_tcp}, stderr: {stderr_tcp}. " + "UDP return code: {returncode_upd}, stdout: {stdout_upd}, stderr: {stderr_upd}." ) else: return "No public IP available to check CUPs version" From 148f055129a43aff35e22500392f3236178715d9 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Mon, 14 Oct 2024 13:42:44 +0200 Subject: [PATCH 11/12] avoid duplicated check_CUPS call --- fedcloud_vm_monitoring/site_monitor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index 44040d0..95d4a32 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -299,7 +299,6 @@ def process_vm(self, vm): for net, addrs in vm["Networks"].items(): vm_ips.extend(addrs) sshd_version = self.get_sshd_version(vm_ips) - CUPS_check = self.check_CUPS(vm_ips) created = parse(vm_info["created_at"]) elapsed = self.now - created output = [ From ee6d297d9eb5115cc01cdd0ddf7b409963ad1843 Mon Sep 17 00:00:00 2001 From: Sebastian Luna-Valero Date: Mon, 14 Oct 2024 13:50:00 +0200 Subject: [PATCH 12/12] remove redundant code --- fedcloud_vm_monitoring/site_monitor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fedcloud_vm_monitoring/site_monitor.py b/fedcloud_vm_monitoring/site_monitor.py index 95d4a32..87ee784 100644 --- a/fedcloud_vm_monitoring/site_monitor.py +++ b/fedcloud_vm_monitoring/site_monitor.py @@ -250,10 +250,7 @@ def _run_shell_command(self, command): capture_output=True, text=True, ) - returncode = completed.returncode - stdout = completed.stdout - stderr = completed.stderr - return returncode, stdout, stderr + return completed.returncode, completed.stdout, completed.stderr def check_open_port(self, ip, port, protocol): if protocol == "tcp":