Merge branch 'v3' of github.com:jumpserver/jumpserver into v3

pull/9123/head
ibuler 2022-11-25 23:11:28 +08:00
commit 5e503ec5b8
19 changed files with 314 additions and 137 deletions

View File

@ -1,5 +1,5 @@
# ~*~ coding: utf-8 ~*~
from django.db.models import F
from django.views.generic.detail import SingleObjectMixin
from django.utils.translation import ugettext as _
from rest_framework.views import APIView, Response
@ -29,6 +29,7 @@ class DomainViewSet(OrgBulkModelViewSet):
class GatewayViewSet(OrgBulkModelViewSet):
perm_model = Host
filterset_fields = ("domain__name", "name", "domain")
search_fields = ("domain__name",)
serializer_class = serializers.GatewaySerializer

View File

@ -207,17 +207,6 @@ class Asset(NodesRelationMixin, AbsConnectivity, JMSOrgBaseModel):
tree_node = TreeNode(**data)
return tree_node
def filter_accounts(self, account_names=None):
from perms.models import AssetPermission
if account_names is None:
return self.accounts.all()
if AssetPermission.SpecialAccount.ALL in account_names:
return self.accounts.all()
# queries = Q(name__in=account_names) | Q(username__in=account_names)
queries = Q(username__in=account_names)
accounts = self.accounts.filter(queries)
return accounts
class Meta:
unique_together = [('org_id', 'name')]
verbose_name = _("Asset")

View File

@ -3,7 +3,6 @@ from .common import Asset
class Host(Asset):
pass
@classmethod
def get_gateway_queryset(cls):

View File

@ -1,23 +1,22 @@
# -*- coding: utf-8 -*-
#
import io
import os
import sshpubkeys
from hashlib import md5
from django.db import models
import sshpubkeys
from django.conf import settings
from django.utils import timezone
from django.db import models
from django.db.models import QuerySet
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from assets.const import Connectivity, SecretType
from common.db import fields
from common.utils import (
ssh_key_string_to_obj, ssh_key_gen, get_logger,
random_string, ssh_pubkey_gen, lazyproperty
random_string, lazyproperty, parse_ssh_public_key_str
)
from common.db import fields
from orgs.mixins.models import JMSOrgBaseModel
from assets.const import Connectivity, SecretType
logger = get_logger(__file__)
@ -62,6 +61,10 @@ class BaseAccount(JMSOrgBaseModel):
def has_secret(self):
return bool(self.secret)
@property
def has_username(self):
return bool(self.username)
@property
def specific(self):
data = {}
@ -84,7 +87,7 @@ class BaseAccount(JMSOrgBaseModel):
@lazyproperty
def public_key(self):
if self.secret_type == SecretType.SSH_KEY:
return ssh_pubkey_gen(private_key=self.private_key)
return parse_ssh_public_key_str(self.private_key)
return None
@property
@ -93,7 +96,7 @@ class BaseAccount(JMSOrgBaseModel):
public_key = self.public_key
elif self.private_key:
try:
public_key = ssh_pubkey_gen(private_key=self.private_key)
public_key = parse_ssh_public_key_str(self.private_key)
except IOError as e:
return str(e)
else:
@ -125,12 +128,9 @@ class BaseAccount(JMSOrgBaseModel):
return key_path
def get_private_key(self):
if not self.private_key_obj:
if not self.private_key:
return None
string_io = io.StringIO()
self.private_key_obj.write_private_key(string_io)
private_key = string_io.getvalue()
return private_key
return self.private_key
@property
def public_key_obj(self):

View File

@ -183,7 +183,7 @@ class CommandFilterRule(OrgModelMixin):
cls, user_id=None, user_group_id=None, account=None,
asset_id=None, org_id=None
):
from perms.models.const import SpecialAccount
from assets.models import Account
user_groups = []
user = get_object_or_none(User, pk=user_id)
if user:
@ -202,7 +202,7 @@ class CommandFilterRule(OrgModelMixin):
if account:
org_id = account.org_id
q |= Q(accounts__contains=account.username) | \
Q(accounts__contains=SpecialAccount.ALL.value)
Q(accounts__contains=Account.AliasAccount.ALL)
if asset:
org_id = asset.org_id
q |= Q(assets=asset)

View File

@ -13,8 +13,9 @@ from django.utils.translation import ugettext_lazy as _
from common.db import fields
from common.utils import get_logger, lazyproperty
from orgs.mixins.models import OrgModelMixin
from assets.models import Host
from .base import BaseAccount
from ..const import SecretType, GATEWAY_NAME
from ..const import SecretType
logger = get_logger(__file__)
@ -37,7 +38,7 @@ class Domain(OrgModelMixin):
@lazyproperty
def gateways(self):
return self.assets.filter(platform__name=GATEWAY_NAME, is_active=True)
return Host.get_gateway_queryset().filter(domain=self, is_active=True)
def select_gateway(self):
return self.random_gateway()

View File

@ -1,28 +1,34 @@
# -*- coding: utf-8 -*-
#
from rest_framework import serializers
from rest_framework.generics import get_object_or_404
from django.utils.translation import ugettext_lazy as _
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
from common.drf.serializers import SecretReadableMixin
from ..models import Domain, Asset
from orgs.mixins.serializers import BulkOrgResourceModelSerializer, OrgResourceSerializerMixin
from common.drf.serializers import SecretReadableMixin, WritableNestedModelSerializer
from common.drf.fields import ObjectRelatedField, EncryptedField
from assets.models import Platform, Node
from assets.const import SecretType, GATEWAY_NAME
from ..serializers import AssetProtocolsSerializer
from ..models import Domain, Asset, Account, Host
from .utils import validate_password_for_ansible, validate_ssh_key
class DomainSerializer(BulkOrgResourceModelSerializer):
asset_count = serializers.SerializerMethodField(label=_('Assets amount'))
gateway_count = serializers.SerializerMethodField(label=_('Gateways count'))
assets = ObjectRelatedField(
many=True, required=False, queryset=Asset.objects, label=_('Asset')
)
class Meta:
model = Domain
fields_mini = ['id', 'name']
fields_small = fields_mini + [
'comment', 'date_created'
]
fields_m2m = [
'asset_count', 'assets', 'gateway_count',
]
fields = fields_small + fields_m2m
read_only_fields = ('asset_count', 'gateway_count', 'date_created')
fields_small = fields_mini + ['comment']
fields_m2m = ['assets']
read_only_fields = ['asset_count', 'gateway_count', 'date_created']
fields = fields_small + fields_m2m + read_only_fields
extra_kwargs = {
'assets': {'required': False, 'label': _('Assets')},
}
@ -36,20 +42,110 @@ class DomainSerializer(BulkOrgResourceModelSerializer):
return obj.gateways.count()
class GatewaySerializer(BulkOrgResourceModelSerializer):
is_connective = serializers.BooleanField(required=False, label=_('Connectivity'))
class GatewaySerializer(BulkOrgResourceModelSerializer, WritableNestedModelSerializer):
password = EncryptedField(
label=_('Password'), required=False, allow_blank=True, allow_null=True, max_length=1024,
validators=[validate_password_for_ansible], write_only=True
)
private_key = EncryptedField(
label=_('SSH private key'), required=False, allow_blank=True, allow_null=True,
max_length=16384, write_only=True
)
passphrase = serializers.CharField(
label=_('Key password'), allow_blank=True, allow_null=True, required=False, write_only=True,
max_length=512,
)
username = serializers.CharField(
label=_('Username'), allow_blank=True, max_length=128, required=True, write_only=True
)
username_display = serializers.SerializerMethodField(label=_('Username'))
protocols = AssetProtocolsSerializer(many=True, required=False, label=_('Protocols'))
class Meta:
model = Asset
fields_mini = ['id']
fields_small = fields_mini + [
'address', 'port', 'protocol',
'is_active', 'is_connective',
'date_created', 'date_updated',
'created_by', 'comment',
model = Host
fields_mini = ['id', 'name', 'address']
fields_small = fields_mini + ['is_active', 'comment']
fields = fields_small + ['domain', 'protocols'] + [
'username', 'password', 'private_key', 'passphrase', 'username_display'
]
fields_fk = ['domain']
fields = fields_small + fields_fk
extra_kwargs = {
'name': {'label': _("Name")},
'address': {'label': _('Address')},
}
@staticmethod
def get_username_display(obj):
account = obj.accounts.order_by('-privileged').first()
return account.username if account else ''
def validate_private_key(self, secret):
if not secret:
return
passphrase = self.initial_data.get('passphrase')
passphrase = passphrase if passphrase else None
validate_ssh_key(secret, passphrase)
return secret
@staticmethod
def clean_auth_fields(validated_data):
username = validated_data.pop('username', None)
password = validated_data.pop('password', None)
private_key = validated_data.pop('private_key', None)
validated_data.pop('passphrase', None)
return username, password, private_key
@staticmethod
def generate_default_data():
platform = Platform.objects.get(name=GATEWAY_NAME, internal=True)
# node = Node.objects.all().order_by('date_created').first()
data = {
'platform': platform,
}
return data
@staticmethod
def create_accounts(instance, username, password, private_key):
account_name = f'{instance.name}-{_("Gateway")}'
account_data = {
'privileged': True,
'name': account_name,
'username': username,
'asset_id': instance.id,
'created_by': instance.created_by
}
if password:
Account.objects.create(
**account_data, secret=password, secret_type=SecretType.PASSWORD
)
if private_key:
Account.objects.create(
**account_data, secret=private_key, secret_type=SecretType.SSH_KEY
)
@staticmethod
def update_accounts(instance, username, password, private_key):
accounts = instance.accounts.filter(username=username)
if password:
account = get_object_or_404(accounts, SecretType.PASSWORD)
account.secret = password
account.save()
if private_key:
account = get_object_or_404(accounts, SecretType.SSH_KEY)
account.secret = private_key
account.save()
def create(self, validated_data):
auth_fields = self.clean_auth_fields(validated_data)
validated_data.update(self.generate_default_data())
instance = super().create(validated_data)
self.create_accounts(instance, *auth_fields)
return instance
def update(self, instance, validated_data):
auth_fields = self.clean_auth_fields(validated_data)
instance = super().update(instance, validated_data)
self.update_accounts(instance, *auth_fields)
return instance
class GatewayWithAuthSerializer(SecretReadableMixin, GatewaySerializer):

View File

@ -1,9 +1,7 @@
from io import StringIO
from django.utils.translation import ugettext_lazy as _
from rest_framework import serializers
from common.utils import ssh_private_key_gen, validate_ssh_private_key
from common.utils import validate_ssh_private_key, parse_ssh_private_key_str
def validate_password_for_ansible(password):
@ -24,9 +22,4 @@ def validate_ssh_key(ssh_key, passphrase=None):
valid = validate_ssh_private_key(ssh_key, password=passphrase)
if not valid:
raise serializers.ValidationError(_("private key invalid or passphrase error"))
ssh_key = ssh_private_key_gen(ssh_key, password=passphrase)
string_io = StringIO()
ssh_key.write_private_key(string_io)
ssh_key = string_io.getvalue()
return ssh_key
return parse_ssh_private_key_str(ssh_key, passphrase)

View File

@ -0,0 +1,18 @@
# Generated by Django 3.2.14 on 2022-11-23 02:26
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('authentication', '0014_auto_20221122_2152'),
]
operations = [
migrations.AlterField(
model_name='connectiontoken',
name='login',
field=models.CharField(max_length=128, verbose_name='Login account'),
),
]

View File

@ -1,24 +1,23 @@
# -*- coding: utf-8 -*-
#
import re
import json
from six import string_types
import base64
import os
import time
import hashlib
import json
import os
import re
import time
from io import StringIO
from itertools import chain
import paramiko
import sshpubkeys
from cryptography.hazmat.primitives import serialization
from django.conf import settings
from django.core.serializers.json import DjangoJSONEncoder
from itsdangerous import (
TimedJSONWebSignatureSerializer, JSONWebSignatureSerializer,
BadSignature, SignatureExpired
)
from django.conf import settings
from django.core.serializers.json import DjangoJSONEncoder
from django.db.models.fields.files import FileField
from six import string_types
from .http import http_date
@ -69,22 +68,19 @@ class Signer(metaclass=Singleton):
return None
_supported_paramiko_ssh_key_types = (paramiko.RSAKey, paramiko.DSSKey, paramiko.Ed25519Key)
def ssh_key_string_to_obj(text, password=None):
key = None
try:
key = paramiko.RSAKey.from_private_key(StringIO(text), password=password)
except paramiko.SSHException:
pass
else:
return key
try:
key = paramiko.DSSKey.from_private_key(StringIO(text), password=password)
except paramiko.SSHException:
pass
else:
return key
for ssh_key_type in _supported_paramiko_ssh_key_types:
if not isinstance(ssh_key_type, paramiko.PKey):
continue
try:
key = ssh_key_type.from_private_key(StringIO(text), password=password)
return key
except paramiko.SSHException:
pass
return key
@ -137,17 +133,68 @@ def ssh_key_gen(length=2048, type='rsa', password=None, username='jumpserver', h
def validate_ssh_private_key(text, password=None):
if isinstance(text, bytes):
if isinstance(text, str):
try:
text = text.decode("utf-8")
text = text.encode("utf-8")
except UnicodeDecodeError:
return False
if isinstance(password, str):
try:
password = password.encode("utf-8")
except UnicodeDecodeError:
return False
key = ssh_key_string_to_obj(text, password=password)
if key is None:
return False
else:
return True
key = parse_ssh_private_key_str(text, password=password)
return bool(key)
def parse_ssh_private_key_str(text: bytes, password=None) -> str:
private_key = _parse_ssh_private_key(text, password=password)
if private_key is None:
return ""
private_key_bytes = private_key.private_bytes(serialization.Encoding.PEM,
serialization.PrivateFormat.OpenSSH,
serialization.NoEncryption())
return private_key_bytes.decode('utf-8')
def parse_ssh_public_key_str(text: bytes = "", password=None) -> str:
private_key = _parse_ssh_private_key(text, password=password)
if private_key is None:
return ""
public_key_bytes = private_key.public_key().public_bytes(serialization.Encoding.OpenSSH,
serialization.PublicFormat.OpenSSH)
return public_key_bytes.decode('utf-8')
def _parse_ssh_private_key(text, password=None):
"""
text: bytes
password: str
return:private key types:
ec.EllipticCurvePrivateKey,
rsa.RSAPrivateKey,
dsa.DSAPrivateKey,
ed25519.Ed25519PrivateKey,
"""
if isinstance(text, str):
try:
text = text.encode("utf-8")
except UnicodeDecodeError:
return None
if password is not None:
if isinstance(password, str):
try:
password = password.encode("utf-8")
except UnicodeDecodeError:
return None
try:
private_key = serialization.load_ssh_private_key(text, password=password)
return private_key
except (ValueError, TypeError):
pass
return None
def validate_ssh_public_key(text):

View File

@ -1,4 +1,3 @@
from django.shortcuts import get_object_or_404
from rest_framework import viewsets
from ops.models import Job, JobExecution
@ -7,14 +6,17 @@ from ops.serializers.job import JobSerializer, JobExecutionSerializer
__all__ = ['JobViewSet', 'JobExecutionViewSet']
from ops.tasks import run_ops_job, run_ops_job_executions
from orgs.mixins.api import OrgBulkModelViewSet
class JobViewSet(viewsets.ModelViewSet):
class JobViewSet(OrgBulkModelViewSet):
serializer_class = JobSerializer
queryset = Job.objects.all()
model = Job
permission_classes = ()
def get_queryset(self):
return self.queryset.filter(instant=False)
query_set = super().get_queryset()
return query_set.filter(instant=False)
def perform_create(self, serializer):
instance = serializer.save()
@ -22,20 +24,20 @@ class JobViewSet(viewsets.ModelViewSet):
run_ops_job.delay(instance.id)
class JobExecutionViewSet(viewsets.ModelViewSet):
class JobExecutionViewSet(OrgBulkModelViewSet):
serializer_class = JobExecutionSerializer
queryset = JobExecution.objects.all()
http_method_names = ('get', 'post', 'head', 'options',)
# filter_fields = ('type',)
permission_classes = ()
model = JobExecution
def perform_create(self, serializer):
instance = serializer.save()
run_ops_job_executions.delay(instance.id)
def get_queryset(self):
query_set = super().get_queryset()
job_id = self.request.query_params.get('job_id')
job_type = self.request.query_params.get('type')
if job_id:
self.queryset = self.queryset.filter(job_id=job_id)
if job_type:
self.queryset = self.queryset.filter(job__type=job_type)
return self.queryset
self.queryset = query_set.filter(job_id=job_id)
return query_set

View File

@ -0,0 +1,18 @@
# Generated by Django 3.2.14 on 2022-11-23 09:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('ops', '0033_auto_20221118_1431'),
]
operations = [
migrations.AddField(
model_name='job',
name='org_id',
field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
),
]

View File

@ -0,0 +1,18 @@
# Generated by Django 3.2.14 on 2022-11-23 10:22
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('ops', '0034_job_org_id'),
]
operations = [
migrations.AddField(
model_name='jobexecution',
name='org_id',
field=models.CharField(blank=True, db_index=True, default='', max_length=36, verbose_name='Organization'),
),
]

View File

@ -9,16 +9,14 @@ from django.utils.translation import gettext_lazy as _
from django.utils import timezone
from celery import current_task
from common.const.choices import Trigger
from common.db.models import BaseCreateUpdateModel
__all__ = ["Job", "JobExecution"]
from ops.ansible import JMSInventory, AdHocRunner, PlaybookRunner
from ops.mixin import PeriodTaskModelMixin
from orgs.mixins.models import JMSOrgBaseModel
class Job(BaseCreateUpdateModel, PeriodTaskModelMixin):
class Job(JMSOrgBaseModel, PeriodTaskModelMixin):
class Types(models.TextChoices):
adhoc = 'adhoc', _('Adhoc')
playbook = 'playbook', _('Playbook')
@ -94,7 +92,7 @@ class Job(BaseCreateUpdateModel, PeriodTaskModelMixin):
return self.executions.create()
class JobExecution(BaseCreateUpdateModel):
class JobExecution(JMSOrgBaseModel):
id = models.UUIDField(default=uuid.uuid4, primary_key=True)
task_id = models.UUIDField(null=True)
status = models.CharField(max_length=16, verbose_name=_('Status'), default='running')

View File

@ -1,14 +1,13 @@
from django.db import transaction
from rest_framework import serializers
from common.drf.fields import ReadableHiddenField
from ops.mixin import PeriodTaskSerializerMixin
from ops.models import Job, JobExecution
from orgs.mixins.serializers import BulkOrgResourceModelSerializer
_all_ = []
class JobSerializer(serializers.ModelSerializer, PeriodTaskSerializerMixin):
class JobSerializer(BulkOrgResourceModelSerializer, PeriodTaskSerializerMixin):
owner = ReadableHiddenField(default=serializers.CurrentUserDefault())
class Meta:

View File

@ -10,6 +10,7 @@ from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from common.utils import get_logger, get_object_or_none, get_log_keep_day
from orgs.utils import tmp_to_org
from .celery.decorator import (
register_as_period_task, after_app_shutdown_clean_periodic,
after_app_ready_start
@ -27,28 +28,30 @@ logger = get_logger(__file__)
@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task"))
def run_ops_job(job_id):
job = get_object_or_none(Job, id=job_id)
execution = job.create_execution()
try:
execution.start()
except SoftTimeLimitExceeded:
execution.set_error('Run timeout')
logger.error("Run adhoc timeout")
except Exception as e:
execution.set_error(e)
logger.error("Start adhoc execution error: {}".format(e))
with tmp_to_org(job.org):
execution = job.create_execution()
try:
execution.start()
except SoftTimeLimitExceeded:
execution.set_error('Run timeout')
logger.error("Run adhoc timeout")
except Exception as e:
execution.set_error(e)
logger.error("Start adhoc execution error: {}".format(e))
@shared_task(soft_time_limit=60, queue="ansible", verbose_name=_("Run ansible task execution"))
def run_ops_job_executions(execution_id, **kwargs):
execution = get_object_or_none(JobExecution, id=execution_id)
try:
execution.start()
except SoftTimeLimitExceeded:
execution.set_error('Run timeout')
logger.error("Run adhoc timeout")
except Exception as e:
execution.set_error(e)
logger.error("Start adhoc execution error: {}".format(e))
with tmp_to_org(execution.org):
try:
execution.start()
except SoftTimeLimitExceeded:
execution.set_error('Run timeout')
logger.error("Run adhoc timeout")
except Exception as e:
execution.set_error(e)
logger.error("Start adhoc execution error: {}".format(e))
@shared_task(verbose_name=_('Periodic clear celery tasks'))

View File

@ -125,7 +125,7 @@ class AssetPermission(OrgModelMixin):
"""
asset_ids = self.get_all_assets(flat=True)
q = Q(asset_id__in=asset_ids)
if Account.AliasAccount.ALL in self.accounts:
if Account.AliasAccount.ALL not in self.accounts:
q &= Q(username__in=self.accounts)
accounts = Account.objects.filter(q).order_by('asset__name', 'name', 'username')
if not flat:

View File

@ -12,7 +12,7 @@ from perms.serializers.permission import ActionChoicesField
__all__ = [
'NodeGrantedSerializer', 'AssetGrantedSerializer',
'ActionsSerializer', 'AccountsPermedSerializer'
'AccountsPermedSerializer'
]
@ -43,14 +43,10 @@ class NodeGrantedSerializer(serializers.ModelSerializer):
read_only_fields = fields
class ActionsSerializer(serializers.Serializer):
actions = ActionChoicesField(read_only=True)
class AccountsPermedSerializer(serializers.ModelSerializer):
actions = ActionChoicesField(read_only=True)
class Meta:
model = Account
fields = ['id', 'name', 'username', 'secret_type', 'has_secret', 'actions']
fields = ['id', 'name', 'has_username', 'username', 'has_secret', 'secret_type', 'actions']
read_only_fields = fields

View File

@ -5,5 +5,4 @@ from .user_permission import user_permission_urlpatterns
app_name = 'perms'
urlpatterns = asset_permission_urlpatterns \
+ user_permission_urlpatterns
urlpatterns = asset_permission_urlpatterns + user_permission_urlpatterns