Rewritten linode integration.

- Support for cloud instances as native entities.
- Using Marshmallow dataclasses+schemas instead of custom `Response`
  objects.
- Merge `linode` backend into `linode` plugin.
This commit is contained in:
Fabio Manganiello 2023-03-26 11:23:33 +02:00
parent 4b9c5a0203
commit bc2730c841
Signed by: blacklight
GPG key ID: D90FBA7F76362774
8 changed files with 654 additions and 261 deletions

View file

@ -1,46 +0,0 @@
from typing import Dict, Optional, List
from platypush.backend.sensor import SensorBackend
from platypush.message.event.linode import LinodeInstanceStatusChanged
class LinodeBackend(SensorBackend):
"""
This backend monitors the state of one or more Linode instances.
Triggers:
* :class:`platypush.message.event.linode.LinodeInstanceStatusChanged` when the status of an instance changes.
Requires:
* The :class:`platypush.plugins.linode.LinodePlugin` plugin configured.
"""
def __init__(self, instances: Optional[List[str]] = None, poll_seconds: float = 30.0, **kwargs):
"""
:param instances: List of instances to monitor, by label (default: monitor all the instances).
"""
super().__init__(plugin='linode', poll_seconds=poll_seconds, **kwargs)
self.instances = set(instances or [])
def process_data(self, data: Dict[str, dict], new_data: Optional[Dict[str, dict]] = None, **kwargs):
instances = data.get('instances', {})
old_instances = (self.data or {}).get('instances', {})
if self.instances:
instances = {label: instances[label] for label in self.instances if label in instances}
if not instances:
return
for label, instance in instances.items():
old_instance = old_instances.get(label, {})
if 'status' in old_instance and old_instance['status'] != instance['status']:
self.bus.post(LinodeInstanceStatusChanged(instance=label,
status=instance['status'],
old_status=old_instance['status']))
# vim:sw=4:ts=4:et:

View file

@ -1,8 +0,0 @@
manifest:
events:
platypush.message.event.linode.LinodeInstanceStatusChanged: when the status of
an instance changes.
install:
pip: []
package: platypush.backend.linode
type: backend

View file

@ -0,0 +1,43 @@
from sqlalchemy import (
Column,
ForeignKey,
Integer,
JSON,
String,
)
from platypush.common.db import Base
from .devices import Device
if 'cloud_instance' not in Base.metadata:
class CloudInstance(Device):
"""
Entity that maps a cloud node - like a Linode or AWS instance.
"""
__tablename__ = 'cloud_instance'
id = Column(
Integer, ForeignKey(Device.id, ondelete='CASCADE'), primary_key=True
)
status = Column(String)
instance_type = Column(String)
ipv4_addresses = Column(JSON)
ipv6_address = Column(String)
group = Column(String)
tags = Column(JSON)
image = Column(String)
region = Column(String)
hypervisor = Column(String)
uuid = Column(String)
specs = Column(JSON)
alerts = Column(JSON)
backups = Column(JSON)
__mapper_args__ = {
'polymorphic_identity': __tablename__,
}

View file

@ -4,15 +4,39 @@ from platypush.message.event import Event
class LinodeEvent(Event): class LinodeEvent(Event):
pass """
Base Linode event class.
"""
class LinodeInstanceStatusChanged(LinodeEvent): class LinodeInstanceStatusChanged(LinodeEvent):
""" """
Event triggered when the status of a Linode instance changes. Event triggered when the status of a Linode instance changes.
""" """
def __init__(self, instance: str, status: str, old_status: Optional[str] = None, *args, **kwargs):
super().__init__(*args, instance=instance, status=status, old_status=old_status, **kwargs) def __init__(
self,
*args,
instance_id: int,
instance: str,
status: str,
old_status: Optional[str] = None,
**kwargs
):
"""
:param instance_id: Linode instance ID.
:param instance: Linode instance name.
:param status: New status of the instance.
:param old_status: Old status of the instance.
"""
super().__init__(
*args,
instance_id=instance_id,
instance=instance,
status=status,
old_status=old_status,
**kwargs
)
# vim:sw=4:ts=4:et: # vim:sw=4:ts=4:et:

View file

@ -1,164 +0,0 @@
import datetime
from typing import List
from linode_api4.objects.linode import Instance, Config, Disk, Backup, Image, Kernel, Type
from platypush.message import Mapping
from platypush.message.response import Response
class LinodeResponse(Response):
pass
class LinodeConfigModel(Mapping):
def __init__(self, config: Config):
super().__init__()
self.comments = config.comments
self.created = config.created
self.helpers = config.helpers.dict
self.id = config.id
self.initrd = config.initrd
self.kernel = dict(LinodeKernelModel(config.kernel))
self.label = config.label
self.linode_id = config.linode_id
self.memory_limit = config.memory_limit
self.parent_id_name = config.parent_id_name
self.root_device = config.root_device
self.run_level = config.run_level
self.updated = datetime.datetime.fromisoformat(config.updated)
self.virt_mode = config.virt_mode
class LinodeKernelModel(Mapping):
def __init__(self, kernel: Kernel):
super().__init__()
self.architecture = kernel.architecture
self.created = kernel.created
self.deprecated = kernel.deprecated
self.description = kernel.description
self.id = kernel.id
self.kvm = kernel.kvm
self.label = kernel.label
self.version = kernel.version
self.xen = kernel.xen
class LinodeBackupModel(Mapping):
def __init__(self, backup: Backup):
super().__init__()
self.created = backup.created
self.disks = {
disk.label: {
'label': disk.label,
'size': disk.size,
'filesystem': disk.filesystem,
}
for disk in backup.disks
}
self.duration = backup.duration
self.finished = backup.finished
self.id = backup.id
self.label = backup.label
self.linode_id = backup.linode_id
self.message = backup.message
self.parent_id_name = backup.parent_id_name
self.country = backup.region.country
self.status = backup.status
self.type = backup.type
self.updated = backup.updated
class LinodeDiskModel(Mapping):
def __init__(self, disk: Disk):
super().__init__()
self.created = disk.created
self.filesystem = disk.filesystem
self.id = disk.id
self.label = disk.label
self.linode_id = disk.linode_id
self.parent_id_name = disk.parent_id_name
self.size = disk.size
self.status = disk.status
self.updated = disk.updated
class LinodeImageModel(Mapping):
def __init__(self, image: Image):
super().__init__()
self.created = image.created
self.created_by = image.created_by
self.deprecated = image.deprecated
self.description = image.description
self.is_public = image.is_public
self.label = image.label
self.size = image.size
self.status = image.status
self.type = image.type
self.vendor = image.vendor
class LinodeTypeModel(Mapping):
# noinspection PyShadowingBuiltins
def __init__(self, type: Type):
super().__init__()
self.disk = type.disk
self.id = type.id
self.label = type.label
self.memory = type.memory
self.network_out = type.network_out
self.price = type.price.dict
self.transfer = type.transfer
self.type_class = type.type_class
self.vcpus = type.vcpus
class LinodeInstanceModel(Mapping):
def __init__(self, node: Instance):
super().__init__()
self.label = node.label
self.status = node.status
self.alerts = node.alerts.dict
self.available_backups = [
dict(LinodeBackupModel(backup))
for backup in node.available_backups.automatic
],
self.backups = {
'enabled': node.backups.enabled,
'schedule': node.backups.schedule.dict,
'last_successful': datetime.datetime.fromisoformat(node.backups.last_successful),
}
self.configs = {config.label: dict(LinodeConfigModel(config)) for config in node.configs}
self.disks = {disk.label: dict(LinodeDiskModel(disk)) for disk in node.disks}
self.group = node.group
self.hypervisor = node.hypervisor
self.id = node.id
self.image = LinodeImageModel(node.image)
self.country = node.region.country
self.specs = node.specs.dict
self.tags = node.tags
self.transfer = node.transfer.dict
self.type = dict(LinodeTypeModel(node.type))
self.updated = node.updated
class LinodeInstanceResponse(LinodeResponse):
def __init__(self, instance: Instance, *args, **kwargs):
super().__init__(*args, output={
'instance': dict(LinodeInstanceModel(instance))
}, **kwargs)
class LinodeInstancesResponse(LinodeResponse):
def __init__(self,
instances: List[Instance],
*args, **kwargs):
super().__init__(*args, output={
'instances': {instance.label: dict(LinodeInstanceModel(instance)) for instance in instances},
}, **kwargs)
# vim:sw=4:ts=4:et:

View file

@ -1,13 +1,22 @@
from typing import Optional, Union from typing import Collection, Dict, List, Optional
from linode_api4 import LinodeClient, Instance from typing_extensions import override
from platypush.plugins.sensor import SensorPlugin
from platypush.message.response.linode import LinodeInstancesResponse, LinodeInstanceResponse from linode_api4 import LinodeClient, Instance, objects
from platypush.plugins import action
from platypush.context import get_bus
from platypush.entities.cloud import CloudInstance
from platypush.message.event.linode import LinodeInstanceStatusChanged
from platypush.schemas.linode import (
LinodeInstance,
LinodeInstanceSchema,
LinodeInstanceStatus,
)
from platypush.entities.managers.cloud import CloudInstanceEntityManager, InstanceId
from platypush.plugins import RunnablePlugin, action
class LinodePlugin(SensorPlugin): class LinodePlugin(RunnablePlugin, CloudInstanceEntityManager):
""" """
This plugin can interact with a Linode account and manage node and volumes. This plugin can interact with a Linode account and manage node and volumes.
@ -21,79 +30,192 @@ class LinodePlugin(SensorPlugin):
* **linode_api4** (``pip install linode_api4``) * **linode_api4** (``pip install linode_api4``)
Triggers:
* :class:`platypush.message.event.linode.LinodeInstanceStatusChanged` when the status of an instance changes.
""" """
def __init__(self, token: str, **kwargs): def __init__(self, token: str, poll_interval: float = 10.0, **kwargs):
""" """
:param token: Your Linode token. :param token: Linode API token.
:param poll_interval: How often to poll the Linode API
(default: 60 seconds).
""" """
super().__init__(poll_interval=poll_interval, **kwargs)
super().__init__(**kwargs)
self._token = token self._token = token
self._instances: Dict[int, CloudInstance] = {}
""" ``{instance_id: CloudInstance}`` mapping. """
def _get_client(self, token: Optional[str] = None) -> LinodeClient: def _get_client(self, token: Optional[str] = None) -> LinodeClient:
"""
Get a :class:`LinodeClient` instance.
:param token: Override the default token.
"""
return LinodeClient(token or self._token) return LinodeClient(token or self._token)
def _get_instance(self, label: str, token: Optional[str] = None) -> Instance: def _get_instance(
self, instance: InstanceId, token: Optional[str] = None
) -> Instance:
"""
Get an instance by name or ID.
:param instance: The label, ID or host UUID of the instance.
:param token: Override the default token.
"""
client = self._get_client(token) client = self._get_client(token)
instances = client.linode.instances(Instance.label == label) if isinstance(instance, str):
assert instances, 'No such Linode instance: ' + label filters = Instance.label == instance
elif isinstance(instance, int):
filters = Instance.id == instance
else:
raise AssertionError(f'Invalid instance type: {type(instance)}')
instances = client.linode.instances(*filters)
assert instances, f'No such Linode instance: {instance}'
return instances[0] return instances[0]
def _linode_instance_to_dict(self, instance: Instance) -> dict:
"""
Convert an internal :class:`linode_api4.Instance` to a
dictionary representation that can be used to create a
:class:`platypush.entities.cloud.CloudInstance` object.
"""
return {
key: (value.dict if isinstance(value, objects.MappedObject) else value)
for key, value in instance.__dict__.items()
if not key.startswith('_')
}
@override
def main(self):
while not self.should_stop():
status = self._instances.copy()
new_status = self.status(publish_entities=False).output
changed_instances = (
[
instance
for instance in new_status
if not (
status.get(instance.id)
and status[instance.id].status == instance.status
)
]
if new_status
else []
)
if changed_instances:
for instance in changed_instances:
get_bus().post(
LinodeInstanceStatusChanged(
instance_id=instance.id,
instance=instance.name,
status=instance.status,
old_status=(
status[instance.id].status
if status.get(instance.id)
else None
),
)
)
self.publish_entities(changed_instances)
self._instances = new_status
self.wait_stop(self.poll_interval)
@override
def transform_entities(
self, entities: Collection[LinodeInstance]
) -> Collection[CloudInstance]:
schema = LinodeInstanceSchema()
return super().transform_entities(
[
CloudInstance(
reachable=instance.status == LinodeInstanceStatus.RUNNING,
**schema.dump(instance),
)
for instance in entities
]
)
@action @action
def status(self, token: Optional[str] = None, instance: Optional[str] = None) \ @override
-> Union[LinodeInstanceResponse, LinodeInstancesResponse]: def status(
self,
*_,
instance: Optional[InstanceId] = None,
token: Optional[str] = None,
publish_entities: bool = True,
**__,
) -> List[LinodeInstance]:
""" """
Get the full status and info of the instances associated to a selected account. Get the full status and info of the instances associated to a selected account.
:param token: Override the default access token if you want to query another account. :param token: Override the default access token if you want to query another account.
:param instance: Select only one node by label. :param instance: Select only one instance, either by name, ID or host UUID.
:return: :class:`platypush.message.response.linode.LinodeInstanceResponse` if ``label`` is specified, :param publish_entities: Whether
:class:`platypush.message.response.linode.LinodeInstancesResponse` otherwise. :class:`platypush.message.event.entities.EntityUpdateEvent` should
be published for all the instances, whether or not their status has
changed (default: ``True``).
:return: .. schema:: linode.LinodeInstanceSchema(many=True)
""" """
if instance: instances = (
instance = self._get_instance(label=instance) [self._get_instance(instance=instance)]
return LinodeInstanceResponse(instance=instance) if instance
else [
instance
for page in self._get_client(token).linode.instances().lists
for instance in page
]
)
client = self._get_client(token) mapped_instances = LinodeInstanceSchema(many=True).load(
return LinodeInstancesResponse(instances=client.linode.instances()) map(self._linode_instance_to_dict, instances)
)
if publish_entities:
self.publish_entities(mapped_instances)
return mapped_instances
@override
@action @action
def reboot(self, instance: str, token: Optional[str] = None) -> None: def reboot(self, instance: InstanceId, token: Optional[str] = None, **_):
""" """
Reboot an instance. Reboot an instance.
:param instance: Label of the instance to be rebooted. :param instance: Instance ID, label or host UUID.
:param token: Default access token override. :param token: Default access token override.
""" """
instance = self._get_instance(label=instance, token=token) node = self._get_instance(instance=instance, token=token)
assert instance.reboot(), 'Reboot failed' assert node.reboot(), 'Reboot failed'
@override
@action @action
def boot(self, instance: str, token: Optional[str] = None) -> None: def boot(self, instance: InstanceId, token: Optional[str] = None, **_):
""" """
Boot an instance. Boot an instance.
:param instance: Label of the instance to be booted. :param instance: Instance ID, label or host UUID.
:param token: Default access token override. :param token: Default access token override.
""" """
instance = self._get_instance(label=instance, token=token) node = self._get_instance(instance=instance, token=token)
assert instance.boot(), 'Boot failed' assert node.boot(), 'Boot failed'
@override
@action @action
def shutdown(self, instance: str, token: Optional[str] = None) -> None: def shutdown(self, instance: InstanceId, token: Optional[str] = None, **_):
""" """
Shutdown an instance. Shutdown an instance.
:param instance: Label of the instance to be shut down. :param instance: Instance ID, label or host UUID.
:param token: Default access token override. :param token: Default access token override.
""" """
instance = self._get_instance(label=instance, token=token) node = self._get_instance(instance=instance, token=token)
assert instance.shutdown(), 'Shutdown failed' assert node.shutdown(), 'Shutdown failed'
@action
def get_measurement(self, *args, **kwargs):
return self.status(*args, **kwargs)
# vim:sw=4:ts=4:et: # vim:sw=4:ts=4:et:

View file

@ -1,5 +1,6 @@
manifest: manifest:
events: {} events:
platypush.message.event.linode.LinodeInstanceStatusChanged:
install: install:
pip: pip:
- linode_api4 - linode_api4

421
platypush/schemas/linode.py Normal file
View file

@ -0,0 +1,421 @@
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from functools import partial
from typing import Any, List, Optional
from marshmallow import pre_load
from marshmallow.fields import Function
from marshmallow.validate import Range
from marshmallow_dataclass import class_schema
from platypush.schemas import EnumField
from platypush.schemas.dataclasses import DataClassSchema
class LinodeInstanceStatus(Enum):
"""
Maps the possible states of an instance.
"""
RUNNING = 'running'
OFFLINE = 'offline'
BOOTING = 'booting'
REBOOTING = 'rebooting'
SHUTTING_DOWN = 'shutting_down'
PROVISIONING = 'provisioning'
DELETING = 'deleting'
MIGRATING = 'migrating'
REBUILDING = 'rebuilding'
CLONING = 'cloning'
RESTORING = 'restoring'
STOPPED = 'stopped'
class LinodeInstanceBackupScheduleDay(Enum):
"""
Allowed values for ``backups.schedule.day``.
"""
SCHEDULING = 'Scheduling'
SUNDAY = 'Sunday'
MONDAY = 'Monday'
TUESDAY = 'Tuesday'
WEDNESDAY = 'Wednesday'
THURSDAY = 'Thursday'
FRIDAY = 'Friday'
SATURDAY = 'Saturday'
class LinodeInstanceBackupScheduleWindow(Enum):
"""
Allowed values for ``backups.schedule.window``.
The window in which your backups will be taken, in UTC. A backups window is
a two-hour span of time in which the backup may occur.
For example, W10 indicates that your backups should be taken between 10:00
and 12:00.
"""
SCHEDULING = 'Scheduling'
W0 = 'W0'
W2 = 'W2'
W4 = 'W4'
W6 = 'W6'
W8 = 'W8'
W10 = 'W10'
W12 = 'W12'
W14 = 'W14'
W16 = 'W16'
W18 = 'W18'
W20 = 'W20'
W22 = 'W22'
class FieldWithId(Function):
"""
Field that handles values that are objects with an ``id`` attribute.
"""
def _deserialize(self, value: Any, *_, **__) -> Optional[Any]:
return value.id if value is not None else None
def _serialize(self, value: Any, *_, **__) -> Optional[Any]:
return value
class LinodeBaseSchema(DataClassSchema):
"""
Base schema for all Linode objects.
"""
TYPE_MAPPING = {
LinodeInstanceStatus: partial( # type: ignore
EnumField, type=LinodeInstanceStatus
),
LinodeInstanceBackupScheduleDay: partial( # type: ignore
EnumField, type=LinodeInstanceBackupScheduleDay
),
LinodeInstanceBackupScheduleWindow: partial( # type: ignore
EnumField, type=LinodeInstanceBackupScheduleWindow
),
**DataClassSchema.TYPE_MAPPING,
}
@pre_load
def pre_load(self, data: dict, **_) -> dict:
from linode_api4.objects.base import MappedObject
# Expand MappedObjects to dictionaries
for key, value in data.items():
if isinstance(value, MappedObject):
data[key] = value.dict
# NOTE Workaround for type -> instance_type not being correctly mapped
if 'type' in data:
data['instance_type'] = data.pop('type')
return data
@dataclass
class LinodeInstanceSpecs:
"""
Class that models the specifications of a Linode instance.
"""
disk: int = field(
metadata={
'metadata': {
'description': 'Allocated disk size, in MB',
'example': 100000,
}
}
)
memory: int = field(
metadata={
'metadata': {
'description': 'Allocated RAM size, in MB',
'example': 8192,
}
}
)
cpus: int = field(
metadata={
'data_key': 'vcpus',
'metadata': {
'description': 'Number of virtual CPUs allocated to the instance',
'example': 4,
},
}
)
gpus: int = field(
metadata={
'metadata': {
'description': 'Number of GPUs allocated to the instance',
'example': 1,
}
}
)
transfer: int = field(
metadata={
'metadata': {
'description': (
'Number of network transfers this instance is allotted each month',
),
'example': 5000,
}
}
)
@dataclass
class LinodeInstanceAlerts:
"""
Class that models the alerts configuration of a Linode instance.
"""
cpu: int = field(
metadata={
'metadata': {
'validate': Range(min=0, max=100),
'description': (
'The percentage of CPU average usage over the past two hours '
'required to trigger an alert',
),
'example': 90,
}
}
)
io: int = field(
metadata={
'metadata': {
'description': (
'The amount of disk I/O operations per second required to '
'trigger an alert'
),
'example': 5000,
}
}
)
network_in: int = field(
metadata={
'metadata': {
'description': (
'The amount of incoming network traffic, in Mbit/s, '
'required to trigger an alert'
),
'example': 10,
}
}
)
network_out: int = field(
metadata={
'metadata': {
'description': (
'The amount of outgoing network traffic, in Mbit/s, '
'required to trigger an alert'
),
'example': 10,
}
}
)
transfer_quota: int = field(
metadata={
'metadata': {
'validate': Range(min=0, max=100),
'description': (
'The percentage of network transfer that may be used before '
'an alert is triggered',
),
'example': 80,
}
}
)
@dataclass
class LinodeInstanceBackupSchedule:
"""
Class that models the backup schedule of a Linode instance.
"""
day: Optional[LinodeInstanceBackupScheduleDay]
window: Optional[LinodeInstanceBackupScheduleWindow]
@dataclass
class LinodeInstanceBackups:
"""
Class that models the backup status of a Linode instance.
"""
available: bool
enabled: bool = field(
metadata={
'metadata': {
'description': 'Whether the backups are enabled on this instance',
'example': True,
}
}
)
schedule: LinodeInstanceBackupSchedule
last_successful: Optional[datetime] = field(
metadata={
'metadata': {
'description': 'When the last backup was successful',
'example': '2020-01-01T00:00:00Z',
}
}
)
@dataclass
class LinodeInstance:
"""
Class that models a Linode instance.
"""
id: int = field(
metadata={
'required': True,
'metadata': {
'description': 'Instance ID',
'example': 12345,
},
}
)
name: str = field(
metadata={
'required': True,
'data_key': 'label',
'metadata': {
'description': 'Instance name',
'example': 'my-instance',
},
},
)
instance_type: str = field(
metadata={
'marshmallow_field': FieldWithId(),
'metadata': {
'description': 'Instance type',
'example': 'g6-standard-4',
},
}
)
ipv4_addresses: List[str] = field(
metadata={
'data_key': 'ipv4',
'metadata': {
'description': 'List of IPv4 addresses associated with this instance',
'example': '["1.2.3.4"]',
},
}
)
ipv6_address: str = field(
metadata={
'data_key': 'ipv6',
'metadata': {
'description': 'IPv6 address associated with this instance',
'example': '1234:5678::9abc:def0:1234:5678/128',
},
}
)
group: str = field(
metadata={
'metadata': {
'description': 'Group the instance belongs to',
'example': 'my-group',
}
}
)
status: LinodeInstanceStatus = field(
metadata={
'metadata': {
'description': 'Instance status',
'example': 'running',
}
}
)
tags: List[str] = field(
metadata={
'metadata': {
'description': 'List of tags associated with this instance',
'example': '["tag1", "tag2"]',
}
}
)
image: str = field(
metadata={
'marshmallow_field': FieldWithId(),
'metadata': {
'description': 'Image used to ',
'example': 'linode/archlinux2014.04',
},
}
)
region: str = field(
metadata={
'marshmallow_field': FieldWithId(),
'metadata': {
'description': 'Region where the instance is located',
'example': 'eu-west',
},
}
)
hypervisor: str = field(
metadata={
'metadata': {
'description': 'The virtualization engine powering this instance',
'example': 'kvm',
}
}
)
specs: LinodeInstanceSpecs
alerts: LinodeInstanceAlerts
backups: LinodeInstanceBackups
created_at: datetime = field(
metadata={
'data_key': 'created',
'metadata': {
'description': 'Instance creation date',
'example': '2020-01-01T00:00:00Z',
},
}
)
updated_at: datetime = field(
metadata={
'data_key': 'updated',
'metadata': {
'description': 'When the instance was last polled/updated',
'example': '2020-01-01T01:00:00Z',
},
}
)
LinodeInstanceSchema = class_schema(LinodeInstance, base_schema=LinodeBaseSchema)