Feature/plugin registry cache adjustments

nodescraper/models/analyzerargs.py

@@ -25,7 +25,13 @@
 ###############################################################################
 from typing import Any
 
-from pydantic import BaseModel, model_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    SerializerFunctionWrapHandler,
+    model_serializer,
+    model_validator,
+)
 
 
 class AnalyzerArgs(BaseModel):
@@ -37,7 +43,18 @@ class AnalyzerArgs(BaseModel):
 
     """
 
-    model_config = {"extra": "forbid", "exclude_none": True}
+    model_config = ConfigDict(extra="forbid")
+
+    @model_serializer(mode="wrap")
+    def serialize_model(self, handler: SerializerFunctionWrapHandler) -> dict[str, object]:
+        serialized = handler(self)
+        remove_keys = []
+        for key, value in serialized.items():
+            if value is None:
+                remove_keys.append(key)
+        for key in remove_keys:
+            del serialized[key]
+        return serialized
 
     @model_validator(mode="before")
     @classmethod
@@ -89,5 +106,5 @@ def build_from_model(cls, datamodel):
             NotImplementedError: Not implemented error
         """
         raise NotImplementedError(
-            "Setting analyzer args from datamodel is not implemented for class: %s", cls.__name__
+            f"Setting analyzer args from datamodel is not implemented for class: {cls.__name__}",
         )

nodescraper/pluginrecipe/__init__.py

@@ -8,15 +8,6 @@
 from nodescraper.models import PluginConfig
 
 from .all_plugins import AllPlugins
-from .discovery import (
-    load_plugin_class,
-    plugin_has_analyzer,
-    plugin_has_collector,
-    plugin_names_matching,
-    plugins_with_analyzer,
-    plugins_with_collector,
-    registered_plugin_names,
-)
 from .node_status import NodeStatus
 from .pluginrecipe import (
     ANALYZE_ONLY,
@@ -40,12 +31,5 @@
     "PluginConfig",
     "PluginRecipe",
     "PluginRunFlags",
-    "load_plugin_class",
     "merge_plugin_configs",
-    "plugin_has_analyzer",
-    "plugin_has_collector",
-    "plugin_names_matching",
-    "plugins_with_analyzer",
-    "plugins_with_collector",
-    "registered_plugin_names",
 ]

nodescraper/pluginrecipe/all_plugins.py

@@ -7,7 +7,7 @@
 ###############################################################################
 from __future__ import annotations
 
-from .discovery import registered_plugin_names
+from .discovery import PluginDiscovery
 from .pluginrecipe import PluginRecipe
 
 
@@ -21,4 +21,4 @@ def plugin_names(cls) -> tuple[str, ...]:
         Returns:
             tuple[str, ...]: Sorted names of all plugins in the plugin registry.
         """
-        return registered_plugin_names()
+        return PluginDiscovery().registered_plugin_names()

nodescraper/pluginrecipe/discovery.py

@@ -4,105 +4,153 @@
 #
 # Copyright (c) 2025 Advanced Micro Devices, Inc.
 #
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
 ###############################################################################
-from __future__ import annotations
-
-from typing import Iterable
-
-
-def registered_plugin_names() -> tuple[str, ...]:
-    """Return all plugin names known to :class:`~nodescraper.pluginregistry.PluginRegistry`.
-
-    Returns:
-        tuple[str, ...]: Sorted registered plugin names.
-    """
-    from nodescraper.pluginregistry import PluginRegistry
-
-    return tuple(sorted(PluginRegistry().plugins))
-
-
-def plugin_names_matching(names: Iterable[str]) -> tuple[str, ...]:
-    """Return plugin names from ``names`` that are registered at runtime.
-
-    Args:
-        names (Iterable[str]): Candidate plugin names to filter.
-
-    Returns:
-        tuple[str, ...]: Sorted subset of ``names`` present in the plugin registry.
-    """
-    available = set(registered_plugin_names())
-    return tuple(sorted(name for name in names if name in available))
 
+from __future__ import annotations
 
-def load_plugin_class(plugin_name: str) -> type | None:
-    """Return a registered plugin class by name.
-
-    Args:
-        plugin_name (str): Registered plugin name.
-
-    Returns:
-        type | None: Plugin class, or ``None`` if the name is not registered.
-    """
-    from nodescraper.pluginregistry import PluginRegistry
-
-    return PluginRegistry().plugins.get(plugin_name)
-
-
-def plugin_has_collector(plugin_name: str) -> bool:
-    """Return whether the plugin exposes a collector task.
-
-    Args:
-        plugin_name (str): Registered plugin name.
+import threading
+from typing import TYPE_CHECKING, Iterable
 
-    Returns:
-        bool: ``True`` when the plugin class defines ``COLLECTOR``.
-    """
-    plugin_class = load_plugin_class(plugin_name)
-    if plugin_class is None:
-        return False
-    collectors = getattr(plugin_class, "get_collector_classes", None)
-    if callable(collectors):
-        return bool(collectors())
-    collector = getattr(plugin_class, "COLLECTOR", None)
-    if collector is None:
-        return False
-    if isinstance(collector, (tuple, list)):
-        return len(collector) > 0
-    return True
-
-
-def plugin_has_analyzer(plugin_name: str) -> bool:
-    """Return whether the plugin exposes an analyzer task.
-
-    Args:
-        plugin_name (str): Registered plugin name.
-
-    Returns:
-        bool: ``True`` when the plugin class defines ``ANALYZER``.
-    """
-    plugin_class = load_plugin_class(plugin_name)
-    return plugin_class is not None and getattr(plugin_class, "ANALYZER", None) is not None
+from nodescraper.pluginregistry import PluginRegistry
 
+if TYPE_CHECKING:
+    from nodescraper.interfaces import PluginInterface
 
-def plugins_with_collector(plugin_names: Iterable[str]) -> tuple[str, ...]:
-    """Filter plugin names to those that define a collector.
 
-    Args:
-        plugin_names (Iterable[str]): Candidate plugin names.
+class PluginDiscovery:
+    """Allows for the discovery of plugins and their capabilities. These external plugins must be
+    registered with the :class:`~nodescraper.pluginregistry.PluginRegistry` before they can be discovered.
 
-    Returns:
-        tuple[str, ...]: Sorted plugin names with a ``COLLECTOR`` implementation.
+    This class can use a cache to avoid repeated PluginRegistry lookups, which can be expensive.
+    If use_cache is False, it will query the PluginRegistry each time.
     """
-    return tuple(sorted(name for name in plugin_names if plugin_has_collector(name)))
-
 
-def plugins_with_analyzer(plugin_names: Iterable[str]) -> tuple[str, ...]:
-    """Filter plugin names to those that define an analyzer.
-
-    Args:
-        plugin_names (Iterable[str]): Candidate plugin names.
-
-    Returns:
-        tuple[str, ...]: Sorted plugin names with an ``ANALYZER`` implementation.
-    """
-    return tuple(sorted(name for name in plugin_names if plugin_has_analyzer(name)))
+    _plugin_cache: dict[str, type[PluginInterface]] | None = None
+    _cache_lock = threading.Lock()
+    COLLECTOR_ATTRIBUTE = "COLLECTOR"
+    ANALYZER_ATTRIBUTE = "ANALYZER"
+
+    def __init__(self, use_cache: bool = True) -> None:
+        """Initialize the PluginDiscovery instance.
+
+        Args:
+            use_cache: If True, cache plugin lookups to improve performance. Defaults to True.
+        """
+        self._use_cache = use_cache
+
+    def load_plugin_class(self, plugin_name: str) -> type | None:
+        if not self._use_cache:
+            return PluginRegistry().plugins.get(plugin_name)
+
+        if PluginDiscovery._plugin_cache is None:
+            with PluginDiscovery._cache_lock:
+                if PluginDiscovery._plugin_cache is None:
+                    PluginDiscovery._plugin_cache = PluginRegistry().plugins
+
+        return PluginDiscovery._plugin_cache.get(plugin_name)
+
+    def plugin_has_collector(self, plugin_name: str) -> bool:
+        """Check if a plugin has a COLLECTOR attribute.
+
+        Args:
+            plugin_name: The name of the plugin to check.
+
+        Returns:
+            True if the plugin exists and has a COLLECTOR attribute, False otherwise.
+        """
+        plugin_class = self.load_plugin_class(plugin_name)
+        return (
+            plugin_class is not None
+            and getattr(plugin_class, self.COLLECTOR_ATTRIBUTE, None) is not None
+        )
+
+    def plugin_has_analyzer(self, plugin_name: str) -> bool:
+        """Check if a plugin has an ANALYZER attribute.
+
+        Args:
+            plugin_name: The name of the plugin to check.
+
+        Returns:
+            True if the plugin exists and has an ANALYZER attribute, False otherwise.
+        """
+        plugin_class = self.load_plugin_class(plugin_name)
+        return (
+            plugin_class is not None
+            and getattr(plugin_class, self.ANALYZER_ATTRIBUTE, None) is not None
+        )
+
+    def plugins_with_collector(self, plugin_names: Iterable[str]) -> tuple[str, ...]:
+        """Filter a list of plugin names to those that have a COLLECTOR attribute.
+
+        Args:
+            plugin_names: An iterable of plugin names to filter.
+
+        Returns:
+            A sorted tuple of plugin names that have a COLLECTOR attribute.
+        """
+        return tuple(sorted(name for name in plugin_names if self.plugin_has_collector(name)))
+
+    def plugins_with_analyzer(self, plugin_names: Iterable[str]) -> tuple[str, ...]:
+        """Filter a list of plugin names to those that have an ANALYZER attribute.
+
+        Args:
+            plugin_names: An iterable of plugin names to filter.
+
+        Returns:
+            A sorted tuple of plugin names that have an ANALYZER attribute.
+        """
+        return tuple(sorted(name for name in plugin_names if self.plugin_has_analyzer(name)))
+
+    @staticmethod
+    def clear_cache() -> None:
+        """Clears the plugin cache, forcing future lookups to query the PluginRegistry again.
+
+        Thread-safe: Acquires the cache lock to ensure no other thread is accessing the cache.
+        """
+        with PluginDiscovery._cache_lock:
+            PluginDiscovery._plugin_cache = None
+
+    def registered_plugin_names(self) -> tuple[str, ...]:
+        """Return all plugin names known to :class:`~nodescraper.pluginregistry.PluginRegistry`.
+
+        Returns:
+            tuple[str, ...]: Sorted registered plugin names.
+        """
+        if not self._use_cache:
+            return tuple(sorted(PluginRegistry().plugins.keys()))
+
+        if PluginDiscovery._plugin_cache is None:
+            with PluginDiscovery._cache_lock:
+                if PluginDiscovery._plugin_cache is None:
+                    PluginDiscovery._plugin_cache = PluginRegistry().plugins
+        return tuple(sorted(PluginDiscovery._plugin_cache.keys()))
+
+    def plugin_names_matching(self, names: Iterable[str]) -> tuple[str, ...]:
+        """Return plugin names from ``names`` that are registered at runtime.
+
+        Args:
+            names (Iterable[str]): Candidate plugin names to filter.
+
+        Returns:
+            tuple[str, ...]: Sorted subset of ``names`` present in the plugin registry.
+        """
+        available = set(self.registered_plugin_names())
+        return tuple(sorted(name for name in names if name in available))

nodescraper/pluginrecipe/node_status.py

@@ -7,7 +7,7 @@
 ###############################################################################
 from __future__ import annotations
 
-from .discovery import plugin_names_matching
+from .discovery import PluginDiscovery
 from .pluginrecipe import PluginRecipe
 
 _NODE_STATUS_PLUGINS = (
@@ -35,4 +35,4 @@ def plugin_names(cls) -> tuple[str, ...]:
         Returns:
             tuple[str, ...]: Sorted node-status plugin names registered in the plugin registry.
         """
-        return plugin_names_matching(_NODE_STATUS_PLUGINS)
+        return PluginDiscovery().plugin_names_matching(_NODE_STATUS_PLUGINS)