diff --git a/apps/assets/forms.py b/apps/assets/forms.py index 0001eeea0..8fc51bc87 100644 --- a/apps/assets/forms.py +++ b/apps/assets/forms.py @@ -32,9 +32,33 @@ class AssetCreateForm(forms.ModelForm): model = Asset tags = forms.ModelMultipleChoiceField(queryset=Tag.objects.all()) fields = [ - 'hostname', 'ip', 'port', 'type', 'comment', 'admin_user', 'idc', 'groups', - 'other_ip', 'remote_card_ip', 'mac_address', 'brand', 'cpu', 'memory', 'disk', 'os', 'cabinet_no', - 'cabinet_pos', 'number', 'status', 'env', 'sn', 'tags', + 'hostname', 'ip', 'public_ip', 'port', 'type', 'comment', 'admin_user', + 'idc', 'groups', 'status', 'env', 'tags', 'is_active' + ] + widgets = { + 'groups': forms.SelectMultiple(attrs={'class': 'select2', + 'data-placeholder': _('Select asset groups')}), + 'tags': forms.SelectMultiple(attrs={'class': 'select2', + 'data-placeholder': _('Select asset tags')}), + 'admin_user': forms.Select(attrs={'class': 'select2', 'data-placeholder': _('Select asset admin user')}), + } + help_texts = { + 'hostname': '* required', + 'ip': '* required', + 'system_users': _('System user will be granted for user to login assets (using ansible create automatic)'), + 'admin_user': _('Admin user should be exist on asset already, And have sudo ALL permission'), + 'tags': '最多5个标签，单个标签最长8个汉字，按回车确认' + } + + +class AssetUpdateForm(AssetCreateForm): + class Meta: + model = Asset + tags = forms.ModelMultipleChoiceField(queryset=Tag.objects.all()) + fields = [ + 'hostname', 'ip', 'port', 'groups', 'admin_user', 'idc', 'is_active', + 'type', 'env', 'status', 'public_ip', 'remote_card_ip', 'cabinet_no', + 'cabinet_pos', 'number', 'comment', 'tags' ] widgets = { 'groups': forms.SelectMultiple(attrs={'class': 'select2', diff --git a/apps/assets/models/asset.py b/apps/assets/models/asset.py index 7386b23bc..068f41aa2 100644 --- a/apps/assets/models/asset.py +++ b/apps/assets/models/asset.py @@ -37,9 +37,8 @@ class Asset(models.Model): ('Test', 'Testing'), ) + # Important ip = models.GenericIPAddressField(max_length=32, verbose_name=_('IP'), db_index=True) - other_ip = models.CharField(max_length=255, null=True, blank=True, verbose_name=_('Other IP')) - remote_card_ip = models.CharField(max_length=16, null=True, blank=True, verbose_name=_('Remote card IP')) hostname = models.CharField(max_length=128, unique=True, verbose_name=_('Hostname')) port = models.IntegerField(default=22, verbose_name=_('Port')) groups = models.ManyToManyField(AssetGroup, blank=True, related_name='assets', verbose_name=_('Asset groups')) @@ -48,24 +47,40 @@ class Asset(models.Model): system_users = models.ManyToManyField(SystemUser, blank=True, related_name='assets', verbose_name=_("System User")) idc = models.ForeignKey(IDC, blank=True, null=True, related_name='assets', on_delete=models.SET_NULL, verbose_name=_('IDC'),) - mac_address = models.CharField(max_length=20, null=True, blank=True, verbose_name=_("Mac address")) - brand = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Brand')) - cpu = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('CPU')) - memory = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Memory')) - disk = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk')) - os = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('OS')) - cabinet_no = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Cabinet number')) - cabinet_pos = models.IntegerField(null=True, blank=True, verbose_name=_('Cabinet position')) - number = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Asset number')) - status = models.CharField(choices=STATUS_CHOICES, max_length=8, null=True, blank=True, - default='In use', verbose_name=_('Asset status')) + is_active = models.BooleanField(default=True, verbose_name=_('Is active')) type = models.CharField(choices=TYPE_CHOICES, max_length=16, blank=True, null=True, default='Server', verbose_name=_('Asset type'),) env = models.CharField(choices=ENV_CHOICES, max_length=8, blank=True, null=True, default='Prod', verbose_name=_('Asset environment'),) + status = models.CharField(choices=STATUS_CHOICES, max_length=8, null=True, blank=True, + default='In use', verbose_name=_('Asset status')) + + # Some information + public_ip = models.GenericIPAddressField(max_length=32, blank=True, null=True, verbose_name=_('Public IP')) + remote_card_ip = models.CharField(max_length=16, null=True, blank=True, verbose_name=_('Remote control card IP')) + cabinet_no = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Cabinet number')) + cabinet_pos = models.IntegerField(null=True, blank=True, verbose_name=_('Cabinet position')) + number = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Asset number')) + + # Collect + vendor = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Vendor')) + model = models.CharField(max_length=54, null=True, blank=True, verbose_name=_('Model')) sn = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('Serial number')) + + cpu_model = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('CPU model')) + cpu_count = models.IntegerField(null=True, verbose_name=_('CPU count')) + cpu_cores = models.IntegerField(null=True, verbose_name=_('CPU cores')) + memory = models.CharField(max_length=64, null=True, blank=True, verbose_name=_('Memory')) + disk_total = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk total')) + disk_info = models.CharField(max_length=1024, null=True, blank=True, verbose_name=_('Disk info')) + + platform = models.CharField(max_length=128, null=True, blank=True, verbose_name='Platform') + os = models.CharField(max_length=128, null=True, blank=True, verbose_name=_('OS')) + os_version = models.FloatField(null=True, blank=True, verbose_name=_('OS Version')) + os_arch = models.CharField(max_length=16, blank=True, null=True, verbose_name=_('OS Arch')) + hostname_raw = models.CharField(max_length=128, blank=True, null=True, verbose_name=_('Hostname raw')) + created_by = models.CharField(max_length=32, null=True, blank=True, verbose_name=_('Created by')) - is_active = models.BooleanField(default=True, verbose_name=_('Is active')) date_created = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name=_('Date added')) comment = models.TextField(max_length=128, default='', blank=True, verbose_name=_('Comment')) tags = models.ManyToManyField('Tag', related_name='assets', blank=True, verbose_name=_('Tags')) @@ -90,6 +105,7 @@ class Asset(models.Model): def _to_secret_json(self): """Ansible use it create inventory""" return { + 'id': self.id, 'hostname': self.hostname, 'ip': self.ip, 'port': self.port, diff --git a/apps/assets/serializers.py b/apps/assets/serializers.py index a9c9c41a8..aae1c4b13 100644 --- a/apps/assets/serializers.py +++ b/apps/assets/serializers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from django.utils.translation import ugettext_lazy as _ -from rest_framework import viewsets, serializers,generics +from rest_framework import viewsets, serializers, generics from .models import AssetGroup, Asset, IDC, AdminUser, SystemUser, Tag from common.mixins import IDInFilterMixin from rest_framework_bulk import BulkListSerializer, BulkSerializerMixin @@ -139,8 +139,8 @@ class AssetSerializer(BulkSerializerMixin, serializers.ModelSerializer): @staticmethod def get_hardware(obj): - if obj.cpu: - return '%s %s %s' % (obj.cpu, obj.memory, obj.disk) + if obj.cpu_count: + return '{} Core {} {}'.format(obj.cpu_count*obj.cpu_cores, obj.memory, obj.disk_total) else: return '' diff --git a/apps/assets/tasks.py b/apps/assets/tasks.py new file mode 100644 index 000000000..8610deb2e --- /dev/null +++ b/apps/assets/tasks.py @@ -0,0 +1,72 @@ +# ~*~ coding: utf-8 ~*~ +from celery import shared_task +import json + +from ops.tasks import run_AdHoc +from common.utils import get_object_or_none, capacity_convert, sum_capacity +from .models import Asset + + +@shared_task +def update_assets_hardware_info(assets): + task_tuple = ( + ('setup', ''), + ) + task = run_AdHoc.delay(task_tuple, assets, record=False) + summary, result = task.get(timeout=60*10) + for hostname, info in result['contacted'].items(): + if info: + info = info[0]['ansible_facts'] + else: + continue + asset = get_object_or_none(Asset, hostname=hostname) + if not asset: + continue + + ___vendor = info['ansible_system_vendor'] + ___model = info['ansible_product_version'] + ___sn = info['ansible_product_serial'] + + for ___cpu_model in info['ansible_processor']: + if ___cpu_model.endswith('GHz'): + break + else: + ___cpu_model = 'Unknown' + ___cpu_count = info['ansible_processor_count'] + ___cpu_cores = info['ansible_processor_cores'] + ___memory = '%s %s' % capacity_convert('{} MB'.format(info['ansible_memtotal_mb'])) + disk_info = {} + for dev, dev_info in info['ansible_devices'].items(): + if dev_info['removable'] == '0': + disk_info[dev] = dev_info['size'] + ___disk_total = '%s %s' % sum_capacity(disk_info.values()) + ___disk_info = json.dumps(disk_info) + + ___platform = info['ansible_system'] + ___os = info['ansible_distribution'] + ___os_version = float(info['ansible_distribution_version']) + ___os_arch = info['ansible_architecture'] + ___hostname_raw = info['ansible_hostname'] + + for k, v in locals().items(): + if k.startswith('___'): + setattr(asset, k.strip('_'), v) + asset.save() + + +@shared_task(name="asset_test_ping_check") +def asset_test_ping_check(assets): + task_tuple = ( + ('ping', ''), + ) + hoc = AdHocRunner(assets) + result = hoc.run(task_tuple) + return result['contacted'].keys(), result['dark'].keys() + + +def get_assets_hardware_info(assets): + task_tuple = ( + ('setup', ''), + ) + task = run_AdHoc.delay(task_tuple, assets, record=False) + return task \ No newline at end of file diff --git a/apps/assets/templates/assets/asset_create.html b/apps/assets/templates/assets/asset_create.html index 5fbe1b772..a1fa33705 100644 --- a/apps/assets/templates/assets/asset_create.html +++ b/apps/assets/templates/assets/asset_create.html @@ -5,10 +5,16 @@ {% block form %}
+ {% if form.no_field_errors %} +
+ {{ form.non_field_errors }} +
+ {% endif %} {% csrf_token %}

{% trans 'Basic' %}

{{ form.hostname|bootstrap_horizontal }} {{ form.ip|bootstrap_horizontal }} + {{ form.public_ip|bootstrap_horizontal }} {{ form.port|bootstrap_horizontal }} {{ form.type|bootstrap_horizontal }} @@ -25,6 +31,8 @@

{% trans 'Other' %}

{{ form.tags|bootstrap_horizontal }} {{ form.comment|bootstrap_horizontal }} + {{ form.is_active|bootstrap_horizontal }} +
diff --git a/apps/assets/templates/assets/asset_update.html b/apps/assets/templates/assets/asset_update.html index bd9f7bde8..0300315fa 100644 --- a/apps/assets/templates/assets/asset_update.html +++ b/apps/assets/templates/assets/asset_update.html @@ -10,10 +10,16 @@ {% block form %} + {% if form.no_field_errors %} +
+ {{ form.non_field_errors }} +
+ {% endif %} {% csrf_token %}

{% trans 'Basic' %}

{{ form.hostname|bootstrap_horizontal }} {{ form.ip|bootstrap_horizontal }} + {{ form.public_ip|bootstrap_horizontal }} {{ form.port|bootstrap_horizontal }} {{ form.type|bootstrap_horizontal }} @@ -26,21 +32,10 @@

{% trans 'Asset user' %}

{{ form.admin_user|bootstrap_horizontal }} -
-

{% trans 'Hardware' %}

- {{ form.sn|bootstrap_horizontal }} - {{ form.brand|bootstrap_horizontal }} - {{ form.cpu|bootstrap_horizontal }} - {{ form.memory|bootstrap_horizontal }} - {{ form.disk|bootstrap_horizontal }} - {{ form.mac_address|bootstrap_horizontal }} -

{% trans 'Configuration' %}

{{ form.number|bootstrap_horizontal }} - {{ form.other_ip|bootstrap_horizontal }} {{ form.remote_card_ip|bootstrap_horizontal }} - {{ form.os|bootstrap_horizontal }}

{% trans 'Location' %}

@@ -53,6 +48,7 @@ {{ form.env|bootstrap_horizontal }} {{ form.tags|bootstrap_horizontal }} {{ form.comment|bootstrap_horizontal }} + {{ form.is_active|bootstrap_horizontal }}
diff --git a/apps/assets/views.py b/apps/assets/views.py index 75abc50c7..9d5adf464 100644 --- a/apps/assets/views.py +++ b/apps/assets/views.py @@ -60,11 +60,6 @@ class AssetCreateView(AdminUserRequiredMixin, CreateAssetTagsMiXin, CreateView): asset.save() return super(AssetCreateView, self).form_valid(form) - def form_invalid(self, form): - if form.errors.get('__all__'): - form.errors['all'] = form.errors.get('__all__') - return super(AssetCreateView, self).form_invalid(form) - def get_context_data(self, **kwargs): context = { 'app': 'Assets', @@ -104,29 +99,29 @@ class AssetModalCreateView(AdminUserRequiredMixin, CreateAssetTagsMiXin, ListVie class AssetUpdateView(AdminUserRequiredMixin, UpdateAssetTagsMiXin, UpdateView): model = Asset - form_class = forms.AssetCreateForm + form_class = forms.AssetUpdateForm template_name = 'assets/asset_update.html' success_url = reverse_lazy('assets:asset-list') - new_form = '' - assets_ids = '' + # new_form = '' + # assets_ids = '' - def post(self, request, *args, **kwargs): - default_keys = [ - 'csrfmiddlewaretoken', - 'assets_ids', - 'ip', - 'number', - 'hostname', - 'system_users', - 'admin_user', - ] - self.assets_ids = self.request.POST.getlist('assets_ids') - self.new_form = self.request.POST.copy() - for i in default_keys: - if self.new_form.has_key(i): - self.new_form.pop(i) + # def post(self, request, *args, **kwargs): + # default_keys = [ + # 'csrfmiddlewaretoken', + # 'assets_ids', + # 'ip', + # 'number', + # 'hostname', + # 'system_users', + # 'admin_user', + # ] + # self.assets_ids = self.request.POST.getlist('assets_ids') + # self.new_form = self.request.POST.copy() + # for i in default_keys: + # if self.new_form.has_key(i): + # self.new_form.pop(i) - return super(AssetUpdateView, self).post(request, *args, **kwargs) + # return super(AssetUpdateView, self).post(request, *args, **kwargs) def get_context_data(self, **kwargs): context = { @@ -152,13 +147,13 @@ class AssetUpdateView(AdminUserRequiredMixin, UpdateAssetTagsMiXin, UpdateView): #delattr(asset, '"%s" % i') #del asset.i - asset.save() - asset.id = 27 - # asset.created_by = self.request.user.username or 'Admin' - asset.save() - asset.id = 28 - asset.save() - return super(AssetUpdateView, self).form_valid(form) + # asset.save() + # asset.id = 27 + # # asset.created_by = self.request.user.username or 'Admin' + # asset.save() + # asset.id = 28 + # asset.save() + # return super(AssetUpdateView, self).form_valid(form) class AssetDeleteView(DeleteView): diff --git a/apps/common/utils.py b/apps/common/utils.py index 7c0c6c8d4..2a6c78dfb 100644 --- a/apps/common/utils.py +++ b/apps/common/utils.py @@ -330,4 +330,52 @@ def encrypt_password(password): return None +from collections import OrderedDict + + +def capacity_convert(size, expect='auto', rate=1000): + """ + :param cap: '100MB', '1G' + :param expect: 'K, M, G, T + :return: + """ + rate_mapping = ( + ('K', rate), + ('KB', rate), + ('M', rate**2), + ('MB', rate**2), + ('G', rate**3), + ('GB', rate**3), + ('T', rate**4), + ('TB', rate**4), + ) + + rate_mapping = OrderedDict(rate_mapping) + + std_size = 0 # To KB + for unit in rate_mapping: + if size.endswith(unit): + try: + std_size = float(size.strip(unit).strip()) * rate_mapping[unit] + except ValueError: + pass + + if expect == 'auto': + for unit, rate_ in rate_mapping.items(): + if rate > std_size/rate_ > 1: + expect = unit + break + expect_size = std_size / rate_mapping[expect] + return expect_size, expect + + +def sum_capacity(cap_list): + total = 0 + for cap in cap_list: + size, _ = capacity_convert(cap, expect='K') + total += size + total = '{} K'.format(total) + return capacity_convert(total, expect='auto') + + signer = Signer() \ No newline at end of file diff --git a/apps/ops/ansible/__init__.py b/apps/ops/ansible/__init__.py new file mode 100644 index 000000000..d59972354 --- /dev/null +++ b/apps/ops/ansible/__init__.py @@ -0,0 +1,6 @@ +# ~*~ coding: utf-8 ~*~ + +from .callback import * +from .inventory import * +from .runner import * + diff --git a/apps/ops/utils/callback.py b/apps/ops/ansible/callback.py similarity index 84% rename from apps/ops/utils/callback.py rename to apps/ops/ansible/callback.py index 148117991..4103ca0f5 100644 --- a/apps/ops/utils/callback.py +++ b/apps/ops/ansible/callback.py @@ -62,6 +62,36 @@ class AdHocResultCallback(CallbackBase): pass +class SingleAdHocResultCallback(CallbackBase): + """ + AdHoc result Callback + """ + def __init__(self, display=None): + self.result_q = dict(contacted={}, dark={}) + super(SingleAdHocResultCallback, self).__init__(display) + + def gather_result(self, n, res): + self.result_q[n][res._host.name].append(res._result) + + def v2_runner_on_ok(self, result): + self.gather_result("contacted", result) + + def v2_runner_on_failed(self, result, ignore_errors=False): + self.gather_result("dark", result) + + def v2_runner_on_unreachable(self, result): + self.gather_result("dark", result) + + def v2_runner_on_skipped(self, result): + self.gather_result("dark", result) + + def v2_playbook_on_task_start(self, task, is_conditional): + pass + + def v2_playbook_on_play_start(self, play): + pass + + class PlaybookResultCallBack(CallbackBase): """ Custom callback model for handlering the output data of diff --git a/apps/ops/utils/inventory.py b/apps/ops/ansible/inventory.py similarity index 100% rename from apps/ops/utils/inventory.py rename to apps/ops/ansible/inventory.py diff --git a/apps/ops/utils/runner.py b/apps/ops/ansible/runner.py similarity index 99% rename from apps/ops/utils/runner.py rename to apps/ops/ansible/runner.py index 1a1612d91..147531105 100644 --- a/apps/ops/utils/runner.py +++ b/apps/ops/ansible/runner.py @@ -271,9 +271,6 @@ class AdHocRunner(object): return result - - - def test_run(): assets = [ { diff --git a/apps/ops/models.py b/apps/ops/models.py index 105c55382..53f6ce291 100644 --- a/apps/ops/models.py +++ b/apps/ops/models.py @@ -22,7 +22,7 @@ class TaskRecord(models.Model): timedelta = models.FloatField(default=0.0, verbose_name=_('Time'), null=True) is_finished = models.BooleanField(default=False, verbose_name=_('Is finished')) is_success = models.BooleanField(default=False, verbose_name=_('Is success')) - assets = models.TextField(blank=True, null=True, verbose_name=_('Assets for hostname')) # Asset inventory may be change + assets = models.TextField(blank=True, null=True, verbose_name=_('Assets for id')) # Asset inventory may be change _modules_args = models.TextField(blank=True, null=True, verbose_name=_('Task module and args json format')) pattern = models.CharField(max_length=64, default='all', verbose_name=_('Task run pattern')) result = models.TextField(blank=True, null=True, verbose_name=_('Task raw result')) @@ -38,9 +38,9 @@ class TaskRecord(models.Model): @property def assets_json(self): from assets.models import Asset - return [Asset.objects.get(hostname=hostname)._to_secret_json() - for hostname in self.total_assets - if Asset.objects.filter(hostname=hostname)] + return [Asset.objects.get(id=int(id_))._to_secret_json() + for id_ in self.total_assets + if Asset.objects.filter(id=int(id_))] @property def module_args(self): diff --git a/apps/ops/tasks.py b/apps/ops/tasks.py index 992ef8e57..8aca5fcb4 100644 --- a/apps/ops/tasks.py +++ b/apps/ops/tasks.py @@ -1,42 +1,32 @@ # coding: utf-8 from __future__ import absolute_import, unicode_literals + import json import time - -from django.utils import timezone from celery import shared_task +from django.utils import timezone +from assets.models import Asset from common.utils import get_logger, encrypt_password -from .utils.runner import AdHocRunner -from .models import TaskRecord +from ops.ansible.runner import AdHocRunner logger = get_logger(__file__) -@shared_task(name="get_assets_hardware_info") -def get_assets_hardware_info(self, assets): - task_tuple = ( - ('setup', ''), - ) - hoc = AdHocRunner(assets) - return hoc.run(task_tuple) -@shared_task(name="asset_test_ping_check") -def asset_test_ping_check(assets): - task_tuple = ( - ('ping', ''), - ) - hoc = AdHocRunner(assets) - result = hoc.run(task_tuple) - return result['contacted'].keys(), result['dark'].keys() - @shared_task(bind=True) def run_AdHoc(self, task_tuple, assets, - task_name='Ansible AdHoc runner', pattern='all', record=True): + task_name='Ansible AdHoc runner', + pattern='all', record=True): + + if not assets: + logger.warning('Empty assets, runner cancel') + if isinstance(assets[0], Asset): + assets = [asset._to_secret_json() for asset in assets] runner = AdHocRunner(assets) if record: @@ -44,7 +34,7 @@ def run_AdHoc(self, task_tuple, assets, if not TaskRecord.objects.filter(uuid=self.request.id): record = TaskRecord(uuid=self.request.id, name=task_name, - assets=','.join(asset['hostname'] for asset in assets), + assets=','.join(str(asset['id']) for asset in assets), module_args=task_tuple, pattern=pattern) record.save() @@ -67,7 +57,7 @@ def run_AdHoc(self, task_tuple, assets, else: record.is_success = False record.save() - return summary + return summary, result def rerun_AdHoc(uuid): diff --git a/apps/ops/utils/mixins.py b/apps/ops/utils/mixins.py deleted file mode 100644 index 9f925a420..000000000 --- a/apps/ops/utils/mixins.py +++ /dev/null @@ -1,13 +0,0 @@ -# ~*~ coding: utf-8 ~*~ - - -class CreateSudoPrivilegesMixin(object): - - def create_privilege(self): - pass - - -class ListSudoPrivilegesMixin(object): - - def get_all_privilege(self): - pass \ No newline at end of file