From f1b37a1e32e4ea57fe19358405041ded8c8198bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 19 Feb 2025 22:29:49 +0100 Subject: [PATCH 01/10] Add dates to changelog --- Changelog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index a938390..6ec4c67 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,7 +2,7 @@ ## [Unreleased] -## [0.1.1] +## [0.1.1] - 19/02/2025 ### New @@ -12,6 +12,6 @@ - Add eth0 as raw_interface_name. -## [0.1.0] +## [0.1.0] - 19/02/2025 - Initial release. \ No newline at end of file From 20cdbfc49979f2f7c02a8a3ef0ae1769fd3a7058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Mon, 24 Feb 2025 12:10:30 +0100 Subject: [PATCH 02/10] Add PyPi installation instruction --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 6bbbd96..21b030b 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ This script work by pulling VMs information from the PVE API and create/update/d ## Installation +This package is available on PyPi. You can install it using pip. + +``` +$ pip install netbox-pve-sync +``` + ## Configuration ### On NetBox From 685e8ac5897aec1d9289c0fd92b734075cfb2992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Mon, 21 Apr 2025 15:54:26 +0200 Subject: [PATCH 03/10] Monitoring PVE ha/replication --- README.md | 14 ++++++++------ netbox_pve_sync/__init__.py | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 21b030b..666a6ef 100644 --- a/README.md +++ b/README.md @@ -34,17 +34,19 @@ You'll also need to perform a minimal configuration on NetBox: - Create the cluster. - Add the following Custom Fields: -| Name | Object types | Label | Type | -|-----------|-----------------|-----------|---------| -| autostart | Virtual Machine | Autostart | Boolean | -| backup | Virtual Disk | Backup | Boolean | -| dns_name | Prefix | DNS Name | Text | +| Name | Object types | Label | Type | +|------------|-----------------|------------|---------| +| autostart | Virtual Machine | Autostart | Boolean | +| replicated | Virtual Machine | Replicated | Boolean | +| ha | Virtual Machine | Failover | Boolean | +| backup | Virtual Disk | Backup | Boolean | +| dns_name | Prefix | DNS Name | Text | ### On the PVE API You'll need to create a dedicated user (ex: netsync) on your PVE cluster and then create an API token. -The user needs to have access to the VM.Monitor, Pool.Audit, VM.Audit permissions. +The user needs to have access to the VM.Monitor, Pool.Audit, VM.Audit, Sys.Audit permissions. The following env variables will need to be set: diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index e25f6ed..882b503 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -98,7 +98,9 @@ def _process_pve_virtual_machine( _nb_objects: dict, _nb_device: any, _pve_tags: [str], - _pve_virtual_machine: dict + _pve_virtual_machine: dict, + _is_replicated: bool, + _has_ha: bool, ) -> dict: _pve_node_name = _nb_device.name.lower() @@ -133,6 +135,8 @@ def _process_pve_virtual_machine( tags=list(map(lambda _pve_tag_name: _nb_objects['tags'][_pve_tag_name].id, _pve_tags)), custom_fields={ 'autostart': pve_virtual_machine_config.get('onboot') == 1, + 'replicated': _is_replicated, + 'ha': _has_ha, } ) else: @@ -145,6 +149,8 @@ def _process_pve_virtual_machine( nb_virtual_machine.status = 'active' if _pve_virtual_machine['status'] == 'running' else 'offline' nb_virtual_machine.tags = list(map(lambda _pve_tag_name: _nb_objects['tags'][_pve_tag_name].id, _pve_tags)) nb_virtual_machine.custom_fields['autostart'] = pve_virtual_machine_config.get('onboot') == 1 + nb_virtual_machine.custom_fields['replicated'] = _is_replicated + nb_virtual_machine.custom_fields['ha'] = _has_ha nb_virtual_machine.save() # Handle the VM network interfaces @@ -432,8 +438,19 @@ def main(): if 'tags' in pve_vm_resource: pass # TODO: pve_vm_tags[pve_vm_resource['vmid']].append(pve_vm_resource['tags']) + pve_ha_virtual_machine_ids = list( + map( + lambda r: int(r['sid'].split(':')[1]), + filter(lambda r: r['type'] == 'service', pve_api.cluster.ha.status.current.get()) + ) + ) + # Process Proxmox nodes for pve_node in pve_api.nodes.get(): + pve_replicated_virtual_machine_ids = list( + map(lambda r: r['guest'], pve_api.nodes(pve_node['node']).replication.get()) + ) + # This script does not create the hardware devices. nb_device = nb_objects['devices'].get(pve_node['node'].lower()) if nb_device is None: @@ -452,6 +469,8 @@ def main(): nb_device, pve_vm_tags.get(pve_virtual_machine['vmid'], []), pve_virtual_machine, + pve_virtual_machine['vmid'] in pve_replicated_virtual_machine_ids, + pve_virtual_machine['vmid'] in pve_ha_virtual_machine_ids, ) From eb663ee3c67324a57490306344cc3c3491cf0158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Mon, 21 Apr 2025 15:56:17 +0200 Subject: [PATCH 04/10] Release 0.2.0 --- Changelog.md | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 6ec4c67..c0ce759 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [0.2.0] - 21/04/2025 + +### New + +- Monitoring PVE HA/Replication. + ## [0.1.1] - 19/02/2025 ### New diff --git a/pyproject.toml b/pyproject.toml index 51aace3..72095db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-pve-sync" -version = "0.1.1" +version = "0.2.0" authors = [ { name = "Aloïs Micard", email = "alois@micard.lu" }, ] From c892d596f49cb1f89e3529f98726ee7be91d1902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Thu, 1 May 2025 08:49:38 +0200 Subject: [PATCH 05/10] [#7] Improve tag handling Closes: #7 --- Changelog.md | 4 ++++ netbox_pve_sync/__init__.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index c0ce759..57c612a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- [#7] Improve tag handling. + ## [0.2.0] - 21/04/2025 ### New diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index 882b503..906bc41 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -88,6 +88,7 @@ def _process_pve_tags( slug=f'pool-{_pve_pool["poolid"]}'.lower(), description=f'Proxmox pool {_pve_pool["poolid"]}', ) + _nb_objects['tags'][_nb_tag.name] = _nb_tag return _nb_objects @@ -433,7 +434,8 @@ def main(): for pve_vm_resource in pve_api.cluster.resources.get(type='vm'): pve_vm_tags[pve_vm_resource['vmid']] = [] - pve_vm_tags[pve_vm_resource['vmid']].append(f'Pool/{pve_vm_resource["pool"]}') + if 'pool' in pve_vm_resource: + pve_vm_tags[pve_vm_resource['vmid']].append(f'Pool/{pve_vm_resource["pool"]}') if 'tags' in pve_vm_resource: pass # TODO: pve_vm_tags[pve_vm_resource['vmid']].append(pve_vm_resource['tags']) From e7d5f5dd26343c0b18b6444d89e4abafd569c310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Thu, 1 May 2025 08:50:40 +0200 Subject: [PATCH 06/10] Release 0.2.1 --- Changelog.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 57c612a..2ab3653 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [0.2.1] - 01/05/2025 + ### Fixed - [#7] Improve tag handling. diff --git a/pyproject.toml b/pyproject.toml index 72095db..151b8f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-pve-sync" -version = "0.2.0" +version = "0.2.1" authors = [ { name = "Aloïs Micard", email = "alois@micard.lu" }, ] From e0c732043cdbfc8b380d9abce13a2fcb7471e05e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Thu, 1 May 2025 09:16:44 +0200 Subject: [PATCH 07/10] Documentation updated to mention NB_CLUSTER_ID --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 666a6ef..900fa1f 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ The following env variables will need to be set: - **NB_API_URL**: The URL to your NetBox instance. (ex: https://netbox.example.org) - **NB_API_TOKEN**: The token created previously. (ex: f74cb99cf552b7005fd1a616b53efba2ce0c9656) +You can also set the `NB_CLUSTER_ID` env variable in order to indicate the ID of the cluster that will be used in +NetBox. + You'll also need to perform a minimal configuration on NetBox: - Create the physical nodes hosting the cluster. (The name should match the one on Proxmox, so that the script can From c0d2c62bbbf15d352ebae3397843aa245c163d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Fri, 2 May 2025 08:20:54 +0200 Subject: [PATCH 08/10] Use `NB_CLUSTER_ID` even for VM update --- Changelog.md | 4 ++++ netbox_pve_sync/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 2ab3653..6b88a73 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Use `NB_CLUSTER_ID` even for VM update. + ## [0.2.1] - 01/05/2025 ### Fixed diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index 906bc41..de581bd 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -143,7 +143,7 @@ def _process_pve_virtual_machine( else: nb_virtual_machine.name = _pve_virtual_machine['name'] nb_virtual_machine.site = _nb_device.site.id - nb_virtual_machine.cluster = 1 + nb_virtual_machine.cluster = os.environ.get('NB_CLUSTER_ID', 1) nb_virtual_machine.device = _nb_device.id nb_virtual_machine.vcpus = pve_virtual_machine_config['cores'] nb_virtual_machine.memory = int(pve_virtual_machine_config['memory']) From e0a94e68d9f2786ca4ef8506e57c4c0704717923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 7 May 2025 09:10:07 +0200 Subject: [PATCH 09/10] Release 0.2.2 --- Changelog.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 6b88a73..933d873 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [0.2.2] - 07/05/2025 + ### Fixed - Use `NB_CLUSTER_ID` even for VM update. diff --git a/pyproject.toml b/pyproject.toml index 151b8f5..37f5f13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-pve-sync" -version = "0.2.1" +version = "0.2.2" authors = [ { name = "Aloïs Micard", email = "alois@micard.lu" }, ] From 1a0107381bec25345674486a29438f34d5ac3a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Tue, 3 Jun 2025 16:36:12 +0200 Subject: [PATCH 10/10] Update run instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 900fa1f..2911245 100644 --- a/README.md +++ b/README.md @@ -63,5 +63,5 @@ The following env variables will need to be set: You can then execute the script using the following command: ``` -PVE_API_HOST=xx PVE_API_USER=xx PVE_API_TOKEN=xx PVE_API_SECRET=xx NB_API_URL=xx NB_API_TOKEN=xx python3 -m netbox_pve_sync -``` \ No newline at end of file +PVE_API_HOST=xx PVE_API_USER=xx PVE_API_TOKEN=xx PVE_API_SECRET=xx NB_API_URL=xx NB_API_TOKEN=xx nbpxsync +```