diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index c7b5b82..463f7a0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -39,14 +39,6 @@ jobs: with: string: ${{ github.repository }} - - name: set dockerhub repo name - uses: mad9000/actions-find-and-replace-string@5 - id: dockerhub_image - with: - source: ${{ steps.docker_image.outputs.lowercase }} - find: 'bb-ricardo' - replace: ${{ vars.DOCKER_HUB_USERNAME }} - - name: Log in to GitHub Container Registry uses: docker/login-action@v4 with: @@ -71,36 +63,3 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - - name: Log in to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ vars.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - - name: Docker hub meta - id: meta_dhub - uses: docker/metadata-action@v5 - with: - images: docker.io/${{ steps.dockerhub_image.outputs.value }} - tags: | - type=semver,pattern={{version}} - type=ref,event=branch - - - name: Build and push with tags - ${{ steps.dockerhub_image.outputs.value }} - uses: docker/build-push-action@v6 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta_dhub.outputs.tags }} - labels: ${{ steps.meta_dhub.outputs.labels }} - -# - name: push README to Dockerhub -# uses: christian-korneck/update-container-description-action@v1 -# env: -# DOCKER_USER: ${{ vars.DOCKER_HUB_USERNAME }} -# DOCKER_PASS: ${{ secrets.DOCKER_HUB_PASSWORD }} -# with: -# destination_container_repo: ${{ steps.dockerhub_image.outputs.value }} -# provider: dockerhub diff --git a/README.md b/README.md index cd83257..f395b1e 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ usage: netbox-sync.py [-h] [-c settings.ini [settings.ini ...]] [-g] Sync objects from various sources to NetBox -Version: 1.9.0 (2026-10-02) +Version: 1.9.1 (2026-10-05) Project URL: https://github.com/bb-ricardo/netbox-sync options: diff --git a/module/__init__.py b/module/__init__.py index f5e5c9c..138d096 100644 --- a/module/__init__.py +++ b/module/__init__.py @@ -7,8 +7,8 @@ # For a copy, see file LICENSE.txt included in this # repository or visit: . -__version__ = "1.9.0" -__version_date__ = "2026-10-02" +__version__ = "1.9.1" +__version_date__ = "2026-10-05" __author__ = "Ricardo Bartels " __description__ = "NetBox Sync" __license__ = "MIT" diff --git a/module/netbox/connection.py b/module/netbox/connection.py index c4d686f..e32ebf6 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -489,8 +489,7 @@ def query_current_data(self, netbox_objects_to_query=None): cache_this_class = False if self.testing_cache is True and len(cached_nb_data) > 0: - for object_data in cached_nb_data: - self.inventory.add_object(nb_object_class, data=object_data, read_from_netbox=True) + self.inventory.load_from_netbox(nb_object_class, cached_nb_data) # mark this object class as retrieved self.resolved_dependencies.add(nb_object_class) @@ -561,8 +560,7 @@ def query_current_data(self, netbox_objects_to_query=None): log.debug(f"Processing %s returned {nb_object_class.name}%s" % (len(nb_objects), plural(len(nb_objects)))) - for object_data in nb_objects: - self.inventory.add_object(nb_object_class, data=object_data, read_from_netbox=True) + self.inventory.load_from_netbox(nb_object_class, nb_objects) # mark this object class as retrieved self.resolved_dependencies.add(nb_object_class) diff --git a/module/netbox/inventory.py b/module/netbox/inventory.py index 4d13740..af39be8 100644 --- a/module/netbox/inventory.py +++ b/module/netbox/inventory.py @@ -40,6 +40,9 @@ def __new__(cls): def init(self): + # object types read from NetBox in this run + self.loaded_types = set() + for object_type in NetBoxObject.__subclasses__(): self.base_structure[object_type.name] = list() @@ -211,6 +214,23 @@ def add_object(self, object_type, data=None, read_from_netbox=False, source=None return new_object + def load_from_netbox(self, object_type, objects_data): + """ + Adds all objects of $object_type read from NetBox to the inventory. + + Parameters + ---------- + object_type: NetBoxObject subclass + object type which was read from NetBox + objects_data: list + data of every object of this type NetBox returned + """ + + for object_data in objects_data: + self.add_object(object_type, data=object_data, read_from_netbox=True) + + self.loaded_types.add(object_type) + def add_update_object(self, object_type, data=None, read_from_netbox=False, source=None): """ Adds new object or updates existing object with data, based on the content of data. diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 2c05f1b..879740e 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -900,7 +900,8 @@ def resolve_relations(self): if resolved_data is not None: self.data[key] = resolved_data - else: + # a type this run did not read from NetBox is expected to be unresolvable, keep the NetBox value + elif data_type in self.inventory.loaded_types: log.error(f"Problems resolving relation '{key}' for object '{self.get_display_name()}' and " f"value '{data_value}'") diff --git a/pyproject.toml b/pyproject.toml index 0dfaa3f..9c284b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "netbox-sync" -version = "1.9.0" +version = "1.9.1" description = "Sync objects from various sources to NetBox" authors = [ { name = "Ricardo Bartels", email = "ricardo.bartels@telekom.de" } diff --git a/settings-example.ini b/settings-example.ini index b1c71fa..1d78159 100644 --- a/settings-example.ini +++ b/settings-example.ini @@ -1,5 +1,5 @@ ;;; Welcome to the NetBox Sync configuration file. -;;; Version: 1.9.0 (2026-10-02) +;;; Version: 1.9.1 (2026-10-05) ;;; Project URL: https://github.com/bb-ricardo/netbox-sync ; The values in this file override the default values used by the system if a config diff --git a/tests/test_unloaded_relations.py b/tests/test_unloaded_relations.py new file mode 100644 index 0000000..57b94da --- /dev/null +++ b/tests/test_unloaded_relations.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020 - 2026 netbox-sync team. All rights reserved. +# +# netbox-sync.py +# +# This work is licensed under the terms of the MIT license. +# For a copy, see file LICENSE.txt included in this +# repository or visit: . + +""" +A run reads only the object types its sources work with. An object can still point at a +type the run did not read, e.g. the module of an interface in a VMware only setup. Such a +relation stays as NetBox returned it: it is not an error and the sync must not change it. +""" +import logging + +import pytest + +from module.netbox.object_classes import NBDevice, NBInterface, NBMACAddress, NBModule, NBPowerPort +from module.sources import instantiate_sources + +# how NetBox returns the module of a component +NETBOX_MODULE = {"id": 5, "display": "Embedded LOM"} + + +def relation_errors(caplog): + return [record.getMessage() for record in caplog.records + if record.levelno == logging.ERROR and "Problems resolving relation" in record.getMessage()] + + +@pytest.mark.parametrize("component_type", [NBInterface, NBPowerPort]) +def test_relation_to_a_type_the_run_did_not_read_is_kept(inventory, caplog, component_type): + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + inventory.load_from_netbox(NBDevice, [{"id": 1, "name": "esx01"}]) + inventory.load_from_netbox(component_type, [{"id": 10, "name": "port0", "device": {"id": 1}, + "module": dict(NETBOX_MODULE)}]) + + inventory.resolve_relations() + + component = inventory.get_by_id(component_type, nb_id=10) + assert relation_errors(caplog) == [] + assert component.data["device"] is inventory.get_by_id(NBDevice, nb_id=1) + assert component.data["module"] == NETBOX_MODULE + + +def test_relation_to_an_object_the_run_created_is_resolved(inventory, caplog): + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + # a source can create objects of a type it did not read, NetBox then returns them as a reference + mac_address = inventory.add_object(NBMACAddress, data={"mac_address": "00:11:22:33:44:55"}) + mac_address.nb_id = 7 + inventory.load_from_netbox(NBDevice, [{"id": 1, "name": "esx01"}]) + inventory.load_from_netbox(NBInterface, [{"id": 10, "name": "port0", "device": {"id": 1}, + "primary_mac_address": {"id": 7}}]) + + inventory.resolve_relations() + + assert relation_errors(caplog) == [] + assert inventory.get_by_id(NBInterface, nb_id=10).data["primary_mac_address"] is mac_address + + +def test_relation_to_a_missing_object_of_a_read_type_is_an_error(inventory, caplog): + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + inventory.load_from_netbox(NBDevice, [{"id": 1, "name": "esx01"}]) + inventory.load_from_netbox(NBModule, []) + inventory.load_from_netbox(NBInterface, [{"id": 10, "name": "port0", "device": {"id": 1}, + "module": dict(NETBOX_MODULE)}]) + + inventory.resolve_relations() + + assert relation_errors(caplog) == [ + f"Problems resolving relation 'module' for object 'port0 (esx01)' and value '{NETBOX_MODULE}'"] + + +def test_vmware_sync_keeps_the_module_of_host_interfaces(vcsim, inventory, load_config, vmware_settings, caplog): + load_config(vmware_settings) + first_run = instantiate_sources()[0] + inventory.resolve_relations() + first_run.apply() + + # the second run finds the objects of the first one in NetBox, the host interfaces with a module + host_interfaces = [i for i in inventory.get_all_items(NBInterface) if "virtual" not in str(i.data.get("type"))] + if not host_interfaces: + pytest.skip(f"vcsim dump '{vcsim.name}' has no physical host interfaces") + for nb_id, this_object in enumerate(inventory.get_all_items(NBInterface) + inventory.get_all_items(NBDevice), + start=1): + this_object.nb_id = nb_id + this_object.is_new = False + this_object.updated_items = list() + this_object.unset_items = list() + for interface in host_interfaces: + interface.data["module"] = dict(NETBOX_MODULE) + + caplog.set_level(logging.ERROR, logger="NetBox-Sync") + inventory.source_list = [] + load_config(vmware_settings) + second_run = instantiate_sources()[0] + inventory.resolve_relations() + second_run.apply() + + assert relation_errors(caplog) == [] + for interface in host_interfaces: + assert interface.data["module"] == NETBOX_MODULE, interface.get_display_name() + assert "module" not in interface.updated_items + interface.unset_items, interface.get_display_name()