From 596e5a6dd1bfc172afe5173a42b00c2b0dccbbf3 Mon Sep 17 00:00:00 2001 From: BaiJiangJie Date: Mon, 11 Nov 2019 16:41:32 +0800 Subject: [PATCH] =?UTF-8?q?[Update]=20=E9=87=8D=E6=9E=84=20LDAP/AD=20?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/authentication/signals_handlers.py | 2 +- apps/settings/api.py | 163 ++++++--- apps/settings/serializers.py | 1 + apps/settings/tasks/__init__.py | 4 + apps/settings/tasks/ldap.py | 17 + .../settings/_ldap_list_users_modal.html | 84 ++++- .../templates/settings/ldap_setting.html | 27 -- apps/settings/urls/api_urls.py | 3 +- apps/settings/utils.py | 219 ------------ apps/settings/utils/__init__.py | 4 + apps/settings/utils/ldap.py | 336 ++++++++++++++++++ apps/settings/views.py | 2 + apps/users/tasks.py | 24 +- 13 files changed, 569 insertions(+), 317 deletions(-) create mode 100644 apps/settings/tasks/__init__.py create mode 100644 apps/settings/tasks/ldap.py delete mode 100644 apps/settings/utils.py create mode 100644 apps/settings/utils/__init__.py create mode 100644 apps/settings/utils/ldap.py diff --git a/apps/authentication/signals_handlers.py b/apps/authentication/signals_handlers.py index c0b48c61d..b894e0651 100644 --- a/apps/authentication/signals_handlers.py +++ b/apps/authentication/signals_handlers.py @@ -47,7 +47,7 @@ def on_openid_login_success(sender, user=None, request=None, **kwargs): @receiver(populate_user) def on_ldap_create_user(sender, user, ldap_user, **kwargs): - if user and user.username != 'admin': + if user and user.username not in ['admin']: user.source = user.SOURCE_LDAP user.save() diff --git a/apps/settings/api.py b/apps/settings/api.py index 22a295b68..c226c9205 100644 --- a/apps/settings/api.py +++ b/apps/settings/api.py @@ -13,10 +13,16 @@ from django.core.mail import send_mail from django.utils.translation import ugettext_lazy as _ from .models import Setting -from .utils import LDAPUtil +from .utils import ( + LDAPServerUtil, LDAPCacheUtil, LDAPImportUtil, LDAPSyncUtil, + LDAP_USE_CACHE_FLAGS + +) +from .tasks import sync_ldap_user_task from common.permissions import IsOrgAdmin, IsSuperUser from common.utils import get_logger from .serializers import MailTestSerializer, LDAPTestSerializer, LDAPUserSerializer +from users.models import User logger = get_logger(__file__) @@ -67,65 +73,107 @@ class LDAPTestingAPI(APIView): success_message = _("Test ldap success") @staticmethod - def get_ldap_util(serializer): - host = serializer.validated_data["AUTH_LDAP_SERVER_URI"] + def get_ldap_config(serializer): + server_uri = serializer.validated_data["AUTH_LDAP_SERVER_URI"] bind_dn = serializer.validated_data["AUTH_LDAP_BIND_DN"] password = serializer.validated_data["AUTH_LDAP_BIND_PASSWORD"] use_ssl = serializer.validated_data.get("AUTH_LDAP_START_TLS", False) search_ougroup = serializer.validated_data["AUTH_LDAP_SEARCH_OU"] search_filter = serializer.validated_data["AUTH_LDAP_SEARCH_FILTER"] attr_map = serializer.validated_data["AUTH_LDAP_USER_ATTR_MAP"] - try: - attr_map = json.loads(attr_map) - except json.JSONDecodeError: - return Response({"error": "AUTH_LDAP_USER_ATTR_MAP not valid"}, status=401) - - util = LDAPUtil( - use_settings_config=False, server_uri=host, bind_dn=bind_dn, - password=password, use_ssl=use_ssl, - search_ougroup=search_ougroup, search_filter=search_filter, - attr_map=attr_map - ) - return util + config = { + 'server_uri': server_uri, + 'bind_dn': bind_dn, + 'password': password, + 'use_ssl': use_ssl, + 'search_ougroup': search_ougroup, + 'search_filter': search_filter, + 'attr_map': json.loads(attr_map), + } + return config def post(self, request): serializer = self.serializer_class(data=request.data) if not serializer.is_valid(): return Response({"error": str(serializer.errors)}, status=401) - util = self.get_ldap_util(serializer) - + attr_map = serializer.validated_data["AUTH_LDAP_USER_ATTR_MAP"] try: - users = util.search_user_items() + json.loads(attr_map) + except json.JSONDecodeError: + return Response({"error": "AUTH_LDAP_USER_ATTR_MAP not valid"}, status=401) + + config = self.get_ldap_config(serializer) + util = LDAPServerUtil(config=config) + try: + users = util.search() except Exception as e: return Response({"error": str(e)}, status=401) - if len(users) > 0: - return Response({"msg": _("Match {} s users").format(len(users))}) - else: - return Response({"error": "Have user but attr mapping error"}, status=401) + return Response({"msg": _("Match {} s users").format(len(users))}) class LDAPUserListApi(generics.ListAPIView): permission_classes = (IsOrgAdmin,) serializer_class = LDAPUserSerializer + def get_queryset_from_cache(self): + search_value = self.request.query_params.get('search') + users = LDAPCacheUtil().search(search_value=search_value) + return users + + def get_queryset_from_server(self): + search_value = self.request.query_params.get('search') + users = LDAPServerUtil().search(search_value=search_value) + return users + def get_queryset(self): if hasattr(self, 'swagger_fake_view'): return [] - q = self.request.query_params.get('search') - try: - util = LDAPUtil() - extra_filter = util.construct_extra_filter(util.SEARCH_FIELD_ALL, q) - users = util.search_user_items(extra_filter) - except Exception as e: - users = [] - logger.error(e) - # 前端data_table会根据row.id对table.selected值进行操作 - for user in users: - user['id'] = user['username'] + cache_police = self.request.query_params.get('cache_police', True) + if cache_police in LDAP_USE_CACHE_FLAGS: + users = self.get_queryset_from_cache() + else: + users = self.get_queryset_from_server() return users + def list(self, request, *args, **kwargs): + cache_police = self.request.query_params.get('cache_police', True) + # 不是用缓存 + if cache_police not in LDAP_USE_CACHE_FLAGS: + return super().list(request, *args, **kwargs) + + queryset = self.get_queryset() + # 缓存有数据 + if queryset is not None: + return super().list(request, *args, **kwargs) + + sync_util = LDAPSyncUtil() + # 还没有同步任务 + if sync_util.task_no_start: + task = sync_ldap_user_task.delay() + data = {'msg': 'Cache no data, sync task {} started.'.format(task.id)} + return Response(data=data, status=409) + # 同步任务正在执行 + if sync_util.task_is_running: + data = {'msg': 'synchronization is running.'} + return Response(data=data, status=409) + # 同步任务执行结束 + if sync_util.task_is_over: + msg = sync_util.get_task_error_msg() + data = {'msg': 'Synchronization task report error: {}'.format(msg)} + return Response(data=data, status=400) + + return super().list(request, *args, **kwargs) + + @staticmethod + def processing_queryset(queryset): + db_username_list = User.objects.all().values_list('username', flat=True) + for q in queryset: + q['id'] = q['username'] + q['existing'] = q['username'] in db_username_list + return queryset + def sort_queryset(self, queryset): order_by = self.request.query_params.get('order') if not order_by: @@ -138,32 +186,41 @@ class LDAPUserListApi(generics.ListAPIView): queryset = sorted(queryset, key=lambda x: x[order_by], reverse=reverse) return queryset - def list(self, request, *args, **kwargs): - queryset = self.get_queryset() + def filter_queryset(self, queryset): + queryset = self.processing_queryset(queryset) queryset = self.sort_queryset(queryset) - page = self.paginate_queryset(queryset) - if page is not None: - return self.get_paginated_response(page) - return Response(queryset) + return queryset -class LDAPUserSyncAPI(APIView): +class LDAPUserImportAPI(APIView): permission_classes = (IsOrgAdmin,) - def post(self, request): - username_list = request.data.get('username_list', []) - - util = LDAPUtil() - try: - result = util.sync_users(username_list) - except Exception as e: - logger.error(e, exc_info=True) - return Response({'error': str(e)}, status=401) + def get_ldap_users(self): + username_list = self.request.data.get('username_list', []) + cache_police = self.request.query_params.get('cache_police', True) + if cache_police in LDAP_USE_CACHE_FLAGS: + users = LDAPCacheUtil().search(search_users=username_list) else: - msg = _("succeed: {} failed: {} total: {}").format( - result['succeed'], result['failed'], result['total'] - ) - return Response({'msg': msg}) + users = LDAPServerUtil().search(search_users=username_list) + return users + + def post(self, request): + users = self.get_ldap_users() + errors = LDAPImportUtil().perform_import(users) + if errors: + return Response({'Error': errors}, status=401) + return Response({'msg': 'Imported {} users successfully'.format(len(users))}) + + +class LDAPCacheRefreshAPI(generics.RetrieveAPIView): + + def retrieve(self, request, *args, **kwargs): + try: + LDAPSyncUtil().clear_cache() + except Exception as e: + logger.error(str(e)) + return Response(data={'msg': str(e)}, status=400) + return Response(data={'msg': 'success'}) class ReplayStorageCreateAPI(APIView): diff --git a/apps/settings/serializers.py b/apps/settings/serializers.py index eb8a61679..0e2e48fa5 100644 --- a/apps/settings/serializers.py +++ b/apps/settings/serializers.py @@ -25,6 +25,7 @@ class LDAPTestSerializer(serializers.Serializer): class LDAPUserSerializer(serializers.Serializer): id = serializers.CharField() username = serializers.CharField() + name = serializers.CharField() email = serializers.CharField() existing = serializers.BooleanField(read_only=True) diff --git a/apps/settings/tasks/__init__.py b/apps/settings/tasks/__init__.py new file mode 100644 index 000000000..87bc6198f --- /dev/null +++ b/apps/settings/tasks/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +# + +from .ldap import * diff --git a/apps/settings/tasks/ldap.py b/apps/settings/tasks/ldap.py new file mode 100644 index 000000000..60058e03e --- /dev/null +++ b/apps/settings/tasks/ldap.py @@ -0,0 +1,17 @@ +# coding: utf-8 +# + +from celery import shared_task + +from common.utils import get_logger +from ..utils import LDAPSyncUtil + +__all__ = ['sync_ldap_user_task'] + + +logger = get_logger(__file__) + + +@shared_task +def sync_ldap_user_task(): + LDAPSyncUtil().perform_sync() diff --git a/apps/settings/templates/settings/_ldap_list_users_modal.html b/apps/settings/templates/settings/_ldap_list_users_modal.html index dd839eb5b..cc63f72c5 100644 --- a/apps/settings/templates/settings/_ldap_list_users_modal.html +++ b/apps/settings/templates/settings/_ldap_list_users_modal.html @@ -23,6 +23,7 @@
+
{% trans "Refresh cache" %}
@@ -43,8 +44,11 @@ diff --git a/apps/settings/templates/settings/ldap_setting.html b/apps/settings/templates/settings/ldap_setting.html index 694fb66f9..e1d7af7a6 100644 --- a/apps/settings/templates/settings/ldap_setting.html +++ b/apps/settings/templates/settings/ldap_setting.html @@ -109,33 +109,6 @@ $(document).ready(function () { error: error }); }) -.on("click","#btn_ldap_modal_confirm",function () { - var username_list = ldap_users_table.selected; - - if (username_list.length === 0){ - var msg = "{% trans 'User is not currently selected, please check the user you want to import'%}"; - toastr.error(msg); - return - } - - var the_url = "{% url "api-settings:ldap-user-sync" %}"; - - function error(message) { - toastr.error(message) - } - - function success(message) { - toastr.success(message.msg) - } - requestApi({ - url: the_url, - body: JSON.stringify({'username_list':username_list}), - method: "POST", - flash_message: false, - success: success, - error: error - }); - }) {% endblock %} diff --git a/apps/settings/urls/api_urls.py b/apps/settings/urls/api_urls.py index bc2e4731f..ee35be25d 100644 --- a/apps/settings/urls/api_urls.py +++ b/apps/settings/urls/api_urls.py @@ -10,7 +10,8 @@ urlpatterns = [ path('mail/testing/', api.MailTestingAPI.as_view(), name='mail-testing'), path('ldap/testing/', api.LDAPTestingAPI.as_view(), name='ldap-testing'), path('ldap/users/', api.LDAPUserListApi.as_view(), name='ldap-user-list'), - path('ldap/users/sync/', api.LDAPUserSyncAPI.as_view(), name='ldap-user-sync'), + path('ldap/users/import/', api.LDAPUserImportAPI.as_view(), name='ldap-user-import'), + path('ldap/cache/refresh/', api.LDAPCacheRefreshAPI.as_view(), name='ldap-cache-refresh'), path('terminal/replay-storage/create/', api.ReplayStorageCreateAPI.as_view(), name='replay-storage-create'), path('terminal/replay-storage/delete/', api.ReplayStorageDeleteAPI.as_view(), name='replay-storage-delete'), path('terminal/command-storage/create/', api.CommandStorageCreateAPI.as_view(), name='command-storage-create'), diff --git a/apps/settings/utils.py b/apps/settings/utils.py deleted file mode 100644 index 9ecd5d286..000000000 --- a/apps/settings/utils.py +++ /dev/null @@ -1,219 +0,0 @@ -# -*- coding: utf-8 -*- -# - -from ldap3 import Server, Connection -from django.utils.translation import ugettext_lazy as _ - -from users.models import User -from users.utils import construct_user_email -from common.utils import get_logger -from common.const import LDAP_AD_ACCOUNT_DISABLE - -from .models import settings - - -logger = get_logger(__file__) - - -class LDAPOUGroupException(Exception): - pass - - -class LDAPUtil: - _conn = None - - SEARCH_FIELD_ALL = 'all' - SEARCH_FIELD_USERNAME = 'username' - - def __init__(self, use_settings_config=True, server_uri=None, bind_dn=None, - password=None, use_ssl=None, search_ougroup=None, - search_filter=None, attr_map=None, auth_ldap=None): - # config - self.paged_size = settings.AUTH_LDAP_SEARCH_PAGED_SIZE - - if use_settings_config: - self._load_config_from_settings() - else: - self.server_uri = server_uri - self.bind_dn = bind_dn - self.password = password - self.use_ssl = use_ssl - self.search_ougroup = search_ougroup - self.search_filter = search_filter - self.attr_map = attr_map - self.auth_ldap = auth_ldap - - def _load_config_from_settings(self): - self.server_uri = settings.AUTH_LDAP_SERVER_URI - self.bind_dn = settings.AUTH_LDAP_BIND_DN - self.password = settings.AUTH_LDAP_BIND_PASSWORD - self.use_ssl = settings.AUTH_LDAP_START_TLS - self.search_ougroup = settings.AUTH_LDAP_SEARCH_OU - self.search_filter = settings.AUTH_LDAP_SEARCH_FILTER - self.attr_map = settings.AUTH_LDAP_USER_ATTR_MAP - self.auth_ldap = settings.AUTH_LDAP - - @property - def connection(self): - if self._conn is None: - server = Server(self.server_uri, use_ssl=self.use_ssl) - conn = Connection(server, self.bind_dn, self.password) - conn.bind() - self._conn = conn - return self._conn - - @staticmethod - def get_user_by_username(username): - try: - user = User.objects.get(username=username) - except Exception as e: - return None - else: - return user - - def _ldap_entry_to_user_item(self, entry): - user_item = {} - for attr, mapping in self.attr_map.items(): - if not hasattr(entry, mapping): - continue - value = getattr(entry, mapping).value or '' - if mapping.lower() == 'useraccountcontrol' and attr == 'is_active'\ - and value: - value = int(value) & LDAP_AD_ACCOUNT_DISABLE \ - != LDAP_AD_ACCOUNT_DISABLE - user_item[attr] = value - return user_item - - def _search_user_items_ou(self, search_ou, extra_filter=None, cookie=None): - search_filter = self.search_filter % {"user": "*"} - if extra_filter: - search_filter = '(&{}{})'.format(search_filter, extra_filter) - - ok = self.connection.search( - search_ou, search_filter, - attributes=list(self.attr_map.values()), - paged_size=self.paged_size, paged_cookie=cookie - ) - if not ok: - error = _("Search no entry matched in ou {}".format(search_ou)) - raise LDAPOUGroupException(error) - - user_items = [] - for entry in self.connection.entries: - user_item = self._ldap_entry_to_user_item(entry) - user = self.get_user_by_username(user_item['username']) - user_item['existing'] = bool(user) - if user_item in user_items: - continue - user_items.append(user_item) - return user_items - - def _cookie(self): - if self.paged_size is None: - cookie = None - else: - cookie = self.connection.result['controls']['1.2.840.113556.1.4.319']['value']['cookie'] - return cookie - - def search_user_items(self, extra_filter=None): - user_items = [] - logger.info("Search user items") - - for search_ou in str(self.search_ougroup).split("|"): - logger.info("Search user search ou: {}".format(search_ou)) - _user_items = self._search_user_items_ou(search_ou, extra_filter=extra_filter) - user_items.extend(_user_items) - while self._cookie(): - logger.info("Page Search user search ou: {}".format(search_ou)) - _user_items = self._search_user_items_ou(search_ou, extra_filter, self._cookie()) - user_items.extend(_user_items) - logger.info("Search user items end") - return user_items - - def construct_extra_filter(self, field, q): - if not q: - return None - extra_filter = '' - if field == self.SEARCH_FIELD_ALL: - for attr in self.attr_map.values(): - extra_filter += '({}={})'.format(attr, q) - extra_filter = '(|{})'.format(extra_filter) - return extra_filter - - if field == self.SEARCH_FIELD_USERNAME and isinstance(q, list): - attr = self.attr_map.get('username') - for username in q: - extra_filter += '({}={})'.format(attr, username) - extra_filter = '(|{})'.format(extra_filter) - return extra_filter - - def search_filter_user_items(self, username_list): - extra_filter = self.construct_extra_filter( - self.SEARCH_FIELD_USERNAME, username_list - ) - user_items = self.search_user_items(extra_filter) - return user_items - - @staticmethod - def save_user(user, user_item): - for field, value in user_item.items(): - if not hasattr(user, field): - continue - if isinstance(getattr(user, field), bool): - if isinstance(value, str): - value = value.lower() - value = value in ['true', 1, True] - setattr(user, field, value) - user.save() - - def update_user(self, user_item): - user = self.get_user_by_username(user_item['username']) - if user.source != User.SOURCE_LDAP: - msg = _('The user source is not LDAP') - return False, msg - try: - self.save_user(user, user_item) - except Exception as e: - logger.error(e, exc_info=True) - return False, str(e) - else: - return True, None - - def create_user(self, user_item): - user = User(source=User.SOURCE_LDAP) - try: - self.save_user(user, user_item) - except Exception as e: - logger.error(e, exc_info=True) - return False, str(e) - else: - return True, None - - @staticmethod - def construct_user_email(user_item): - username = user_item['username'] - email = user_item.get('email', '') - email = construct_user_email(username, email) - return email - - def create_or_update_users(self, user_items): - succeed = failed = 0 - for user_item in user_items: - exist = user_item.pop('existing', False) - user_item['email'] = self.construct_user_email(user_item) - if not exist: - ok, error = self.create_user(user_item) - else: - ok, error = self.update_user(user_item) - if not ok: - logger.info("Failed User: {}".format(user_item)) - failed += 1 - else: - succeed += 1 - result = {'total': len(user_items), 'succeed': succeed, 'failed': failed} - return result - - def sync_users(self, username_list=None): - user_items = self.search_filter_user_items(username_list) - result = self.create_or_update_users(user_items) - return result diff --git a/apps/settings/utils/__init__.py b/apps/settings/utils/__init__.py new file mode 100644 index 000000000..87bc6198f --- /dev/null +++ b/apps/settings/utils/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +# + +from .ldap import * diff --git a/apps/settings/utils/ldap.py b/apps/settings/utils/ldap.py new file mode 100644 index 000000000..ba3e5a838 --- /dev/null +++ b/apps/settings/utils/ldap.py @@ -0,0 +1,336 @@ +# coding: utf-8 +# + +from ldap3 import Server, Connection +from django.conf import settings +from django.core.cache import cache +from django.utils.translation import ugettext_lazy as _ + +from common.const import LDAP_AD_ACCOUNT_DISABLE +from common.utils import timeit, get_logger +from users.utils import construct_user_email +from users.models import User + +logger = get_logger(__file__) + +__all__ = [ + 'LDAPConfig', 'LDAPServerUtil', 'LDAPCacheUtil', 'LDAPImportUtil', + 'LDAPSyncUtil', 'LDAP_USE_CACHE_FLAGS' +] + +LDAP_USE_CACHE_FLAGS = [1, '1', 'true', 'True', True] + + +class LDAPOUGroupException(Exception): + pass + + +class LDAPConfig(object): + + def __init__(self, config=None): + self.server_uri = None + self.bind_dn = None + self.password = None + self.use_ssl = None + self.search_ougroup = None + self.search_filter = None + self.attr_map = None + if isinstance(config, dict): + self.load_from_config(config) + else: + self.load_from_settings() + + def load_from_config(self, config): + self.server_uri = config.get('server_uri') + self.bind_dn = config.get('bind_dn') + self.password = config.get('password') + self.use_ssl = config.get('use_ssl') + self.search_ougroup = config.get('search_ougroup') + self.search_filter = config.get('search_filter') + self.attr_map = config.get('attr_map') + + def load_from_settings(self): + self.server_uri = settings.AUTH_LDAP_SERVER_URI + self.bind_dn = settings.AUTH_LDAP_BIND_DN + self.password = settings.AUTH_LDAP_BIND_PASSWORD + self.use_ssl = settings.AUTH_LDAP_START_TLS + self.search_ougroup = settings.AUTH_LDAP_SEARCH_OU + self.search_filter = settings.AUTH_LDAP_SEARCH_FILTER + self.attr_map = settings.AUTH_LDAP_USER_ATTR_MAP + + +class LDAPServerUtil(object): + + def __init__(self, config=None): + if isinstance(config, dict): + self.config = LDAPConfig(config=config) + elif isinstance(config, LDAPConfig): + self.config = config + else: + self.config = LDAPConfig() + self._conn = None + self._paged_size = self.get_paged_size() + self.search_users = None + self.search_value = None + + @property + def connection(self): + if self._conn: + return self._conn + server = Server(self.config.server_uri, use_ssl=self.config.use_ssl) + conn = Connection(server, self.config.bind_dn, self.config.password) + conn.bind() + self._conn = conn + return self._conn + + @staticmethod + def get_paged_size(): + paged_size = settings.AUTH_LDAP_SEARCH_PAGED_SIZE + if isinstance(paged_size, int): + return paged_size + return None + + def paged_cookie(self): + if self._paged_size is None: + return None + cookie = self.connection.result['controls']['1.2.840.113556.1.4.319']['value']['cookie'] + return cookie + + def get_search_filter_extra(self): + extra = '' + if self.search_users: + mapping_username = self.config.attr_map.get('username') + for user in self.search_users: + extra += '({}={})'.format(mapping_username, user) + return '(|{})'.format(extra) + if self.search_value: + for attr in self.config.attr_map.values(): + extra += '({}={})'.format(attr, self.search_value) + return '(|{})'.format(extra) + return extra + + def get_search_filter(self): + search_filter = self.config.search_filter % {'user': '*'} + search_filter_extra = self.get_search_filter_extra() + if search_filter_extra: + search_filter = '(&{}{})'.format(search_filter, search_filter_extra) + return search_filter + + def search_user_entries_ou(self, search_ou, paged_cookie=None): + logger.info("Search user entries ou: {}, paged_cookie: {}". + format(search_ou, paged_cookie)) + search_filter = self.get_search_filter() + attributes = list(self.config.attr_map.values()) + ok = self.connection.search( + search_base=search_ou, search_filter=search_filter, + attributes=attributes, paged_size=self._paged_size, + paged_cookie=paged_cookie + ) + if not ok: + error = _("Search no entry matched in ou {}".format(search_ou)) + raise LDAPOUGroupException(error) + + @timeit + def search_user_entries(self): + logger.info("Search user entries") + user_entries = list() + search_ous = str(self.config.search_ougroup).split('|') + for search_ou in search_ous: + self.search_user_entries_ou(search_ou) + user_entries.extend(self.connection.entries) + while self.paged_cookie(): + self.search_user_entries_ou(search_ou, self.paged_cookie()) + user_entries.extend(self.connection.entries) + return user_entries + + def user_entry_to_dict(self, entry): + user = {} + attr_map = self.config.attr_map.items() + for attr, mapping in attr_map: + if not hasattr(entry, mapping): + continue + value = getattr(entry, mapping).value or '' + if attr == 'is_active' and mapping.lower() == 'useraccountcontrol' \ + and value: + value = int(value) & LDAP_AD_ACCOUNT_DISABLE != LDAP_AD_ACCOUNT_DISABLE + user[attr] = value + return user + + @timeit + def user_entries_to_dict(self, user_entries): + users = [] + for user_entry in user_entries: + user = self.user_entry_to_dict(user_entry) + users.append(user) + return users + + @timeit + def search(self, search_users=None, search_value=None): + logger.info("Search ldap users") + self.search_users = search_users + self.search_value = search_value + user_entries = self.search_user_entries() + users = self.user_entries_to_dict(user_entries) + return users + + +class LDAPCacheUtil(object): + CACHE_KEY_USERS = 'CACHE_KEY_LDAP_USERS' + + def __init__(self): + self.search_users = None + self.search_value = None + + def set_users(self, users): + logger.info('Set ldap users to cache, count: {}'.format(len(users))) + cache.set(self.CACHE_KEY_USERS, users, None) + + def get_users(self): + users = cache.get(self.CACHE_KEY_USERS) + logger.info('Get ldap users from cache, count: {}'.format(len(users))) + return users + + def delete_users(self): + logger.info('Delete ldap users from cache') + cache.delete(self.CACHE_KEY_USERS) + + def filter_users(self, users): + if self.search_users: + filter_users = [ + user for user in users + if user['username'] in self.search_users + ] + elif self.search_value: + filter_users = [ + user for user in users + if self.search_value in ','.join(user.values()) + ] + else: + filter_users = users + return filter_users + + def search(self, search_users=None, search_value=None): + self.search_users = search_users + self.search_value = search_value + users = self.get_users() + users = self.filter_users(users) + return users + + +class LDAPSyncUtil(object): + CACHE_KEY_LDAP_USERS_SYNC_TASK_ERROR_MSG = 'CACHE_KEY_LDAP_USERS_SYNC_TASK_ERROR_MSG' + + CACHE_KEY_LDAP_USERS_SYNC_TASK_STATUS = 'CACHE_KEY_LDAP_USERS_SYNC_TASK_STATUS' + TASK_STATUS_IS_RUNNING = 'RUNNING' + TASK_STATUS_IS_OVER = 'OVER' + + def __init__(self): + self.server_util = LDAPServerUtil() + self.cache_util = LDAPCacheUtil() + self.task_error_msg = None + + def clear_cache(self): + logger.info('Clear ldap sync cache') + self.delete_task_status() + self.delete_task_error_msg() + self.cache_util.delete_users() + + @property + def task_no_start(self): + status = self.get_task_status() + return status is None + + @property + def task_is_running(self): + status = self.get_task_status() + return status == self.TASK_STATUS_IS_RUNNING + + @property + def task_is_over(self): + status = self.get_task_status() + return status == self.TASK_STATUS_IS_OVER + + def set_task_status(self, status): + logger.info('Set task status: {}'.format(status)) + cache.set(self.CACHE_KEY_LDAP_USERS_SYNC_TASK_STATUS, status, None) + + def get_task_status(self): + status = cache.get(self.CACHE_KEY_LDAP_USERS_SYNC_TASK_STATUS) + logger.info('Get task status: {}'.format(status)) + return status + + def delete_task_status(self): + logger.info('Delete task status') + cache.delete(self.CACHE_KEY_LDAP_USERS_SYNC_TASK_STATUS) + + def set_task_error_msg(self, error_msg): + logger.info('Set task error msg') + cache.set(self.CACHE_KEY_LDAP_USERS_SYNC_TASK_ERROR_MSG, error_msg, None) + + def get_task_error_msg(self): + logger.info('Get task error msg') + error_msg = cache.get(self.CACHE_KEY_LDAP_USERS_SYNC_TASK_ERROR_MSG) + return error_msg + + def delete_task_error_msg(self): + logger.info('Delete task error msg') + cache.delete(self.CACHE_KEY_LDAP_USERS_SYNC_TASK_ERROR_MSG) + + def pre_sync(self): + self.set_task_status(self.TASK_STATUS_IS_RUNNING) + + def sync(self): + users = self.server_util.search() + self.cache_util.set_users(users) + + def post_sync(self): + self.set_task_status(self.TASK_STATUS_IS_OVER) + + def perform_sync(self): + logger.info('Start perform sync ldap users from server to cache') + self.pre_sync() + try: + self.sync() + except Exception as e: + error_msg = str(e) + logger.error(error_msg) + self.set_task_error_msg(error_msg) + self.post_sync() + logger.info('End perform sync ldap users from server to cache') + + +class LDAPImportUtil(object): + + def __init__(self): + pass + + @staticmethod + def get_user_email(user): + username = user['username'] + email = user['email'] + email = construct_user_email(username, email) + return email + + def update_or_create(self, user): + user['email'] = self.get_user_email(user) + if user['username'] not in ['admin']: + user['source'] = User.SOURCE_LDAP + obj, created = User.objects.update_or_create( + username=user['username'], defaults=user + ) + return obj, created + + def perform_import(self, users): + logger.info('Start perform import ldap users, count: {}'.format(len(users))) + errors = [] + for user in users: + try: + self.update_or_create(user) + except Exception as e: + errors.append({user['username']: str(e)}) + logger.error(e) + logger.info('End perform import ldap users') + return errors + + + diff --git a/apps/settings/views.py b/apps/settings/views.py index a9df717d7..2442f074e 100644 --- a/apps/settings/views.py +++ b/apps/settings/views.py @@ -5,6 +5,7 @@ from django.utils.translation import ugettext as _ from common.permissions import PermissionsMixin, IsSuperUser from common import utils +from .utils import LDAPSyncUtil from .forms import EmailSettingForm, LDAPSettingForm, BasicSettingForm, \ TerminalSettingForm, SecuritySettingForm, EmailContentSettingForm @@ -83,6 +84,7 @@ class LDAPSettingView(PermissionsMixin, TemplateView): form.save() msg = _("Update setting successfully") messages.success(request, msg) + LDAPSyncUtil().clear_cache() return redirect('settings:ldap-setting') else: context = self.get_context_data() diff --git a/apps/users/tasks.py b/apps/users/tasks.py index e0051e939..29355514d 100644 --- a/apps/users/tasks.py +++ b/apps/users/tasks.py @@ -11,7 +11,7 @@ from .models import User from .utils import ( send_password_expiration_reminder_mail, send_user_expiration_reminder_mail ) -from settings.utils import LDAPUtil +from settings.utils import LDAPServerUtil, LDAPImportUtil logger = get_logger(__file__) @@ -70,16 +70,21 @@ def check_user_expired_periodic(): @shared_task -def sync_ldap_user(): - logger.info("Start sync ldap user periodic task") - util = LDAPUtil() - result = util.sync_users() - logger.info("Result: {}".format(result)) +def import_ldap_user(): + logger.info("Start import ldap user task") + util_server = LDAPServerUtil() + util_import = LDAPImportUtil() + users = util_server.search() + errors = util_import.perform_import(users) + if errors: + logger.error("Imported LDAP users errors: {}".format(errors)) + else: + logger.info('Imported {} users successfully'.format(len(users))) @shared_task @after_app_ready_start -def sync_ldap_user_periodic(): +def import_ldap_user_periodic(): if not settings.AUTH_LDAP: return if not settings.AUTH_LDAP_SYNC_IS_PERIODIC: @@ -91,10 +96,9 @@ def sync_ldap_user_periodic(): else: interval = None crontab = settings.AUTH_LDAP_SYNC_CRONTAB - tasks = { - 'sync_ldap_user_periodic': { - 'task': sync_ldap_user.name, + 'import_ldap_user_periodic': { + 'task': import_ldap_user.name, 'interval': interval, 'crontab': crontab, 'enabled': True,