From 0c82c4ecf89349d6d54bc7bf54b32b07dd96146b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 19 Feb 2025 17:42:02 +0100 Subject: [PATCH 01/15] Allow to configure NetBox cluster ID Closes: #5 --- netbox_pve_sync/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index 3ffbf57..19b6a15 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -125,7 +125,7 @@ def _process_pve_virtual_machine( serial=_pve_virtual_machine['vmid'], name=_pve_virtual_machine['name'], site=_nb_device.site.id, - cluster=1, # TODO + cluster=os.environ.get('NB_CLUSTER_ID', 1), device=_nb_device.id, vcpus=pve_virtual_machine_config['cores'], memory=int(pve_virtual_machine_config['memory']), From 7943a36b0a1902721b14d7cd34ceaf41a406fb86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 19 Feb 2025 17:48:22 +0100 Subject: [PATCH 02/15] Add Changelog.md --- Changelog.md | 11 +++++++++++ pyproject.toml | 1 + 2 files changed, 12 insertions(+) create mode 100644 Changelog.md diff --git a/Changelog.md b/Changelog.md new file mode 100644 index 0000000..1e40c29 --- /dev/null +++ b/Changelog.md @@ -0,0 +1,11 @@ +# netbox-pve-sync + +## [Unreleased] + +### New + +- [#5] Allow to configure cluster ID. + +## [0.1.0] + +- Initial release. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 574dd9d..2af1bab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ [project.urls] Homepage = "https://git.creekorful.cloud/creekorful/netbox-pve-sync" Issues = "https://git.creekorful.cloud/creekorful/netbox-pve-sync/issues" +Changelog = "https://git.creekorful.cloud/creekorful/netbox-pve-sync/src/branch/main/CHANGELOG.md" [project.scripts] nbpxsync = "netbox_pve_sync:main" From 287056057708dcaeec4c1bc9ab843a25c0bbc67e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 19 Feb 2025 17:51:49 +0100 Subject: [PATCH 03/15] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2af1bab..9095284 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ [project.urls] Homepage = "https://git.creekorful.cloud/creekorful/netbox-pve-sync" Issues = "https://git.creekorful.cloud/creekorful/netbox-pve-sync/issues" -Changelog = "https://git.creekorful.cloud/creekorful/netbox-pve-sync/src/branch/main/CHANGELOG.md" +Changelog = "https://git.creekorful.cloud/creekorful/netbox-pve-sync/src/branch/main/Changelog.md" [project.scripts] nbpxsync = "netbox_pve_sync:main" From d11c1677e2f15d7438d4a48242a5f5038e293f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 19 Feb 2025 22:27:08 +0100 Subject: [PATCH 04/15] Add eth0 as raw_interface_name --- Changelog.md | 4 ++++ netbox_pve_sync/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 1e40c29..61a7504 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,10 @@ - [#5] Allow to configure cluster ID. +### Changed + +- Add eth0 as raw_interface_name. + ## [0.1.0] - Initial release. \ No newline at end of file diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index 19b6a15..e25f6ed 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -246,7 +246,7 @@ def _process_pve_virtual_machine_network_interface( # TODO: Improve Multiple IP address handling _pve_virtual_machine_ip_address = None - for raw_interface_name in ['ens18', 'ens19']: + for raw_interface_name in ['eth0', 'ens18', 'ens19']: if raw_interface_name in _pve_virtual_machine_ip_addresses: _pve_virtual_machine_ip_address = _pve_virtual_machine_ip_addresses[raw_interface_name][0] break From 2f17021e5d6785e5c077d230ef40e5d463a451ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 19 Feb 2025 22:29:18 +0100 Subject: [PATCH 05/15] Release 0.1.1 --- Changelog.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 61a7504..a938390 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [0.1.1] + ### New - [#5] Allow to configure cluster ID. diff --git a/pyproject.toml b/pyproject.toml index 9095284..51aace3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-pve-sync" -version = "0.1.0" +version = "0.1.1" authors = [ { name = "Aloïs Micard", email = "alois@micard.lu" }, ] From f1b37a1e32e4ea57fe19358405041ded8c8198bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 19 Feb 2025 22:29:49 +0100 Subject: [PATCH 06/15] Add dates to changelog --- Changelog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index a938390..6ec4c67 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,7 +2,7 @@ ## [Unreleased] -## [0.1.1] +## [0.1.1] - 19/02/2025 ### New @@ -12,6 +12,6 @@ - Add eth0 as raw_interface_name. -## [0.1.0] +## [0.1.0] - 19/02/2025 - Initial release. \ No newline at end of file From 20cdbfc49979f2f7c02a8a3ef0ae1769fd3a7058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Mon, 24 Feb 2025 12:10:30 +0100 Subject: [PATCH 07/15] Add PyPi installation instruction --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 6bbbd96..21b030b 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ This script work by pulling VMs information from the PVE API and create/update/d ## Installation +This package is available on PyPi. You can install it using pip. + +``` +$ pip install netbox-pve-sync +``` + ## Configuration ### On NetBox From 685e8ac5897aec1d9289c0fd92b734075cfb2992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Mon, 21 Apr 2025 15:54:26 +0200 Subject: [PATCH 08/15] Monitoring PVE ha/replication --- README.md | 14 ++++++++------ netbox_pve_sync/__init__.py | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 21b030b..666a6ef 100644 --- a/README.md +++ b/README.md @@ -34,17 +34,19 @@ You'll also need to perform a minimal configuration on NetBox: - Create the cluster. - Add the following Custom Fields: -| Name | Object types | Label | Type | -|-----------|-----------------|-----------|---------| -| autostart | Virtual Machine | Autostart | Boolean | -| backup | Virtual Disk | Backup | Boolean | -| dns_name | Prefix | DNS Name | Text | +| Name | Object types | Label | Type | +|------------|-----------------|------------|---------| +| autostart | Virtual Machine | Autostart | Boolean | +| replicated | Virtual Machine | Replicated | Boolean | +| ha | Virtual Machine | Failover | Boolean | +| backup | Virtual Disk | Backup | Boolean | +| dns_name | Prefix | DNS Name | Text | ### On the PVE API You'll need to create a dedicated user (ex: netsync) on your PVE cluster and then create an API token. -The user needs to have access to the VM.Monitor, Pool.Audit, VM.Audit permissions. +The user needs to have access to the VM.Monitor, Pool.Audit, VM.Audit, Sys.Audit permissions. The following env variables will need to be set: diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index e25f6ed..882b503 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -98,7 +98,9 @@ def _process_pve_virtual_machine( _nb_objects: dict, _nb_device: any, _pve_tags: [str], - _pve_virtual_machine: dict + _pve_virtual_machine: dict, + _is_replicated: bool, + _has_ha: bool, ) -> dict: _pve_node_name = _nb_device.name.lower() @@ -133,6 +135,8 @@ def _process_pve_virtual_machine( tags=list(map(lambda _pve_tag_name: _nb_objects['tags'][_pve_tag_name].id, _pve_tags)), custom_fields={ 'autostart': pve_virtual_machine_config.get('onboot') == 1, + 'replicated': _is_replicated, + 'ha': _has_ha, } ) else: @@ -145,6 +149,8 @@ def _process_pve_virtual_machine( nb_virtual_machine.status = 'active' if _pve_virtual_machine['status'] == 'running' else 'offline' nb_virtual_machine.tags = list(map(lambda _pve_tag_name: _nb_objects['tags'][_pve_tag_name].id, _pve_tags)) nb_virtual_machine.custom_fields['autostart'] = pve_virtual_machine_config.get('onboot') == 1 + nb_virtual_machine.custom_fields['replicated'] = _is_replicated + nb_virtual_machine.custom_fields['ha'] = _has_ha nb_virtual_machine.save() # Handle the VM network interfaces @@ -432,8 +438,19 @@ def main(): if 'tags' in pve_vm_resource: pass # TODO: pve_vm_tags[pve_vm_resource['vmid']].append(pve_vm_resource['tags']) + pve_ha_virtual_machine_ids = list( + map( + lambda r: int(r['sid'].split(':')[1]), + filter(lambda r: r['type'] == 'service', pve_api.cluster.ha.status.current.get()) + ) + ) + # Process Proxmox nodes for pve_node in pve_api.nodes.get(): + pve_replicated_virtual_machine_ids = list( + map(lambda r: r['guest'], pve_api.nodes(pve_node['node']).replication.get()) + ) + # This script does not create the hardware devices. nb_device = nb_objects['devices'].get(pve_node['node'].lower()) if nb_device is None: @@ -452,6 +469,8 @@ def main(): nb_device, pve_vm_tags.get(pve_virtual_machine['vmid'], []), pve_virtual_machine, + pve_virtual_machine['vmid'] in pve_replicated_virtual_machine_ids, + pve_virtual_machine['vmid'] in pve_ha_virtual_machine_ids, ) From eb663ee3c67324a57490306344cc3c3491cf0158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Mon, 21 Apr 2025 15:56:17 +0200 Subject: [PATCH 09/15] Release 0.2.0 --- Changelog.md | 6 ++++++ pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 6ec4c67..c0ce759 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [0.2.0] - 21/04/2025 + +### New + +- Monitoring PVE HA/Replication. + ## [0.1.1] - 19/02/2025 ### New diff --git a/pyproject.toml b/pyproject.toml index 51aace3..72095db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-pve-sync" -version = "0.1.1" +version = "0.2.0" authors = [ { name = "Aloïs Micard", email = "alois@micard.lu" }, ] From c892d596f49cb1f89e3529f98726ee7be91d1902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Thu, 1 May 2025 08:49:38 +0200 Subject: [PATCH 10/15] [#7] Improve tag handling Closes: #7 --- Changelog.md | 4 ++++ netbox_pve_sync/__init__.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index c0ce759..57c612a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- [#7] Improve tag handling. + ## [0.2.0] - 21/04/2025 ### New diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index 882b503..906bc41 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -88,6 +88,7 @@ def _process_pve_tags( slug=f'pool-{_pve_pool["poolid"]}'.lower(), description=f'Proxmox pool {_pve_pool["poolid"]}', ) + _nb_objects['tags'][_nb_tag.name] = _nb_tag return _nb_objects @@ -433,7 +434,8 @@ def main(): for pve_vm_resource in pve_api.cluster.resources.get(type='vm'): pve_vm_tags[pve_vm_resource['vmid']] = [] - pve_vm_tags[pve_vm_resource['vmid']].append(f'Pool/{pve_vm_resource["pool"]}') + if 'pool' in pve_vm_resource: + pve_vm_tags[pve_vm_resource['vmid']].append(f'Pool/{pve_vm_resource["pool"]}') if 'tags' in pve_vm_resource: pass # TODO: pve_vm_tags[pve_vm_resource['vmid']].append(pve_vm_resource['tags']) From e7d5f5dd26343c0b18b6444d89e4abafd569c310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Thu, 1 May 2025 08:50:40 +0200 Subject: [PATCH 11/15] Release 0.2.1 --- Changelog.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 57c612a..2ab3653 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [0.2.1] - 01/05/2025 + ### Fixed - [#7] Improve tag handling. diff --git a/pyproject.toml b/pyproject.toml index 72095db..151b8f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-pve-sync" -version = "0.2.0" +version = "0.2.1" authors = [ { name = "Aloïs Micard", email = "alois@micard.lu" }, ] From e0c732043cdbfc8b380d9abce13a2fcb7471e05e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Thu, 1 May 2025 09:16:44 +0200 Subject: [PATCH 12/15] Documentation updated to mention NB_CLUSTER_ID --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 666a6ef..900fa1f 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ The following env variables will need to be set: - **NB_API_URL**: The URL to your NetBox instance. (ex: https://netbox.example.org) - **NB_API_TOKEN**: The token created previously. (ex: f74cb99cf552b7005fd1a616b53efba2ce0c9656) +You can also set the `NB_CLUSTER_ID` env variable in order to indicate the ID of the cluster that will be used in +NetBox. + You'll also need to perform a minimal configuration on NetBox: - Create the physical nodes hosting the cluster. (The name should match the one on Proxmox, so that the script can From c0d2c62bbbf15d352ebae3397843aa245c163d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Fri, 2 May 2025 08:20:54 +0200 Subject: [PATCH 13/15] Use `NB_CLUSTER_ID` even for VM update --- Changelog.md | 4 ++++ netbox_pve_sync/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 2ab3653..6b88a73 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- Use `NB_CLUSTER_ID` even for VM update. + ## [0.2.1] - 01/05/2025 ### Fixed diff --git a/netbox_pve_sync/__init__.py b/netbox_pve_sync/__init__.py index 906bc41..de581bd 100644 --- a/netbox_pve_sync/__init__.py +++ b/netbox_pve_sync/__init__.py @@ -143,7 +143,7 @@ def _process_pve_virtual_machine( else: nb_virtual_machine.name = _pve_virtual_machine['name'] nb_virtual_machine.site = _nb_device.site.id - nb_virtual_machine.cluster = 1 + nb_virtual_machine.cluster = os.environ.get('NB_CLUSTER_ID', 1) nb_virtual_machine.device = _nb_device.id nb_virtual_machine.vcpus = pve_virtual_machine_config['cores'] nb_virtual_machine.memory = int(pve_virtual_machine_config['memory']) From e0a94e68d9f2786ca4ef8506e57c4c0704717923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Wed, 7 May 2025 09:10:07 +0200 Subject: [PATCH 14/15] Release 0.2.2 --- Changelog.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 6b88a73..933d873 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [0.2.2] - 07/05/2025 + ### Fixed - Use `NB_CLUSTER_ID` even for VM update. diff --git a/pyproject.toml b/pyproject.toml index 151b8f5..37f5f13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "netbox-pve-sync" -version = "0.2.1" +version = "0.2.2" authors = [ { name = "Aloïs Micard", email = "alois@micard.lu" }, ] From 1a0107381bec25345674486a29438f34d5ac3a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Tue, 3 Jun 2025 16:36:12 +0200 Subject: [PATCH 15/15] Update run instructions --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 900fa1f..2911245 100644 --- a/README.md +++ b/README.md @@ -63,5 +63,5 @@ The following env variables will need to be set: You can then execute the script using the following command: ``` -PVE_API_HOST=xx PVE_API_USER=xx PVE_API_TOKEN=xx PVE_API_SECRET=xx NB_API_URL=xx NB_API_TOKEN=xx python3 -m netbox_pve_sync -``` \ No newline at end of file +PVE_API_HOST=xx PVE_API_USER=xx PVE_API_TOKEN=xx PVE_API_SECRET=xx NB_API_URL=xx NB_API_TOKEN=xx nbpxsync +```