gitee
/
eiam
mirror of https://gitee.com/topiam/eiam
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

行为审计查询优化

pull/65/head
xiuchen 2023-10-05 11:26:46 +08:00
parent 0a1df68e5e
commit f2ed57a026
9 changed files with 142 additions and 617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
eiam-audit/src/main/java/cn/topiam/employee/audit/configuration/AuditDynamicIndexName.java
View File

@ -1,49 +0,0 @@
/*
* eiam-audit - Employee Identity and Access Management
* Copyright © 2022-Present Jinan Yuanchuang Network Technology Co., Ltd. (support@topiam.cn)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cn.topiam.employee.audit.configuration;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import cn.topiam.employee.support.autoconfiguration.SupportProperties;
import static cn.topiam.employee.support.constant.EiamConstants.DEFAULT_DATE_FORMATTER_PATTERN;
/**
* ElasticsearchConfiguration
*
* @author TopIAM
* Created by support@topiam.cn on 2022/11/3 23:31
*/
public class AuditDynamicIndexName {
private final SupportProperties supportProperties;
public AuditDynamicIndexName(SupportProperties supportProperties) {
this.supportProperties = supportProperties;
}
/**
* 获取索引
*
* @return {@link String}
*/
public String getIndexName() {
return supportProperties.getAudit().getIndexPrefix() + LocalDate.now()
.format(DateTimeFormatter.ofPattern(DEFAULT_DATE_FORMATTER_PATTERN));
}
}

245
eiam-audit/src/main/java/cn/topiam/employee/audit/configuration/AuditElasticsearchConfiguration.java
View File

@ -1,245 +0,0 @@
/*
* eiam-audit - Employee Identity and Access Management
* Copyright © 2022-Present Jinan Yuanchuang Network Technology Co., Ltd. (support@topiam.cn)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package cn.topiam.employee.audit.configuration;
import java.util.Set;
import org.jetbrains.annotations.NotNull;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
import org.springframework.data.convert.ReadingConverter;
import org.springframework.data.convert.WritingConverter;
import org.springframework.data.elasticsearch.core.convert.ElasticsearchCustomConversions;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.Lists;
import cn.topiam.employee.audit.enums.EventStatus;
import cn.topiam.employee.audit.enums.TargetType;
import cn.topiam.employee.audit.event.type.EventType;
import cn.topiam.employee.support.autoconfiguration.SupportProperties;
import cn.topiam.employee.support.exception.TopIamException;
import cn.topiam.employee.support.geo.GeoLocationProvider;
import cn.topiam.employee.support.security.userdetails.UserType;
import cn.topiam.employee.support.util.JsonUtils;
import static cn.topiam.employee.common.geo.maxmind.MaxmindGeoLocationServiceImpl.MAXMIND;
import static cn.topiam.employee.support.security.userdetails.UserType.*;
import static cn.topiam.employee.support.security.userdetails.UserType.UNKNOWN;
/**
* ElasticsearchConfiguration
*
* @author TopIAM
* Created by support@topiam.cn on 2022/11/3 23:31
*/
@Configuration
public class AuditElasticsearchConfiguration {
@Bean
public AuditDynamicIndexName auditDynamicIndexName(SupportProperties supportProperties) {
return new AuditDynamicIndexName(supportProperties);
}
@Bean
public ElasticsearchCustomConversions elasticsearchCustomConversions() {
return new ElasticsearchCustomConversions(
Lists.newArrayList(AuditTypeToStringConverter.INSTANCE,
StringToAuditTypeConverter.INSTANCE, EventStatusToStringConverter.INSTANCE,
StringToEventStatusConverter.INSTANCE, ActorTypeToStringConverter.INSTANCE,
StringToActorTypeConverter.INSTANCE, GeoLocationProviderToStringConverter.INSTANCE,
StringToGeoLocationProviderConverter.INSTANCE, TargetTypeToStringConverter.INSTANCE,
StringToTargetTypeConverter.INSTANCE, StringToSetConverter.INSTANCE,
SetToStringConverter.INSTANCE));
}
@WritingConverter
enum AuditTypeToStringConverter implements Converter<EventType, String> {
/**
* INSTANCE
*/
INSTANCE,;
@Override
public String convert(EventType source) {
return source.getCode();
}
}
@ReadingConverter
enum StringToAuditTypeConverter implements Converter<String, EventType> {
/**
*INSTANCE
*/
INSTANCE;
@Override
public EventType convert(@NotNull String source) {
return EventType.getType(source);
}
}
@WritingConverter
enum ActorTypeToStringConverter implements Converter<UserType, String> {
/**
* INSTANCE
*/
INSTANCE,;
@Override
public String convert(UserType source) {
return source.getType();
}
}
@ReadingConverter
enum StringToActorTypeConverter implements Converter<String, UserType> {
/**
* INSTANCE
*/
INSTANCE,;
@Override
public UserType convert(@NotNull String source) {
if (source.equals(ADMIN.getType())) {
return ADMIN;
}
if (source.equals(USER.getType())) {
return USER;
}
if (source.equals(DEVELOPER.getType())) {
return DEVELOPER;
}
if (source.equals(UNKNOWN.getType())) {
return UNKNOWN;
}
throw new TopIamException("未知用户类型");
}
}
@WritingConverter
enum TargetTypeToStringConverter implements Converter<TargetType, String> {
/**
* INSTANCE
*/
INSTANCE,;
@Override
public String convert(TargetType source) {
return source.getCode();
}
}
@ReadingConverter
enum StringToTargetTypeConverter implements Converter<String, TargetType> {
/**
*INSTANCE
*/
INSTANCE;
@Override
public TargetType convert(@NotNull String source) {
return TargetType.getType(source);
}
}
@WritingConverter
enum GeoLocationProviderToStringConverter implements Converter<GeoLocationProvider, String> {
/**
* INSTANCE
*/
INSTANCE,;
@Override
public String convert(GeoLocationProvider source) {
return source.getProvider();
}
}
@ReadingConverter
enum StringToGeoLocationProviderConverter implements Converter<String, GeoLocationProvider> {
/**
*INSTANCE
*/
INSTANCE;
@Override
public GeoLocationProvider convert(@NotNull String source) {
if (MAXMIND.getProvider().equals(source)) {
return MAXMIND;
}
if (GeoLocationProvider.NONE.getProvider().equals(source)) {
return GeoLocationProvider.NONE;
}
throw new TopIamException("未找到提供商");
}
}
@WritingConverter
enum EventStatusToStringConverter implements Converter<EventStatus, String> {
/**
* INSTANCE
*/
INSTANCE,;
@Override
public String convert(@NotNull EventStatus source) {
return source.getCode();
}
}
@ReadingConverter
enum StringToEventStatusConverter implements Converter<String, EventStatus> {
/**
*INSTANCE
*/
INSTANCE;
@Override
public EventStatus convert(@NotNull String source) {
return EventStatus.getType(source);
}
}
@WritingConverter
enum SetToStringConverter implements Converter<Set<String>, String> {
/**
* INSTANCE
*/
INSTANCE,;
@Override
public String convert(@NotNull Set<String> source) {
return JsonUtils.writeValueAsString(source);
}
}
@ReadingConverter
enum StringToSetConverter implements Converter<String, Set<String>> {
/**
*INSTANCE
*/
INSTANCE;
@Override
public Set<String> convert(@NotNull String source) {
return JsonUtils.readValue(source, new TypeReference<>() {
});
}
}
}

143
eiam-audit/src/main/java/cn/topiam/employee/audit/service/converter/AuditDataConverter.java
View File

@ -17,28 +17,22 @@
*/
package cn.topiam.employee.audit.service.converter;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import org.mapstruct.Mapper;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.client.elc.NativeQuery;
import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
import org.springframework.data.elasticsearch.client.elc.Queries;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import com.google.common.collect.Lists;
import com.querydsl.core.types.ExpressionUtils;
import com.querydsl.core.types.Predicate;
import cn.topiam.employee.audit.controller.pojo.AuditListQuery;
import cn.topiam.employee.audit.controller.pojo.AuditListResult;
import cn.topiam.employee.audit.entity.AuditEntity;
import cn.topiam.employee.audit.entity.QAuditEntity;
import cn.topiam.employee.audit.entity.Target;
import cn.topiam.employee.audit.enums.TargetType;
import cn.topiam.employee.common.entity.account.OrganizationEntity;
@ -61,19 +55,6 @@ import cn.topiam.employee.support.repository.page.domain.Page;
import cn.topiam.employee.support.repository.page.domain.PageModel;
import cn.topiam.employee.support.security.userdetails.UserType;
import co.elastic.clients.elasticsearch._types.FieldSort;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.SortOptions;
import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField;
import co.elastic.clients.json.JsonData;
import static cn.topiam.employee.audit.entity.Actor.ACTOR_ID;
import static cn.topiam.employee.audit.entity.Actor.ACTOR_TYPE;
import static cn.topiam.employee.audit.entity.Event.*;
import static cn.topiam.employee.support.constant.EiamConstants.DEFAULT_DATE_TIME_FORMATTER_PATTERN;
/**
* 审计数据转换
*
@ -87,46 +68,45 @@ public interface AuditDataConverter {
/**
* searchHits 转审计列表
*
* @param search {@link SearchHits}
* @param auditEntityPage {@link Page}
* @param page {@link PageModel}
* @return {@link Page}
*/
default Page<AuditListResult> searchHitsConvertToAuditListResult(SearchHits<AuditEntity> search,
PageModel page) {
default Page<AuditListResult> entityConvertToAuditListResult(org.springframework.data.domain.Page<AuditEntity> auditEntityPage,
PageModel page) {
List<AuditListResult> list = new ArrayList<>();
//总记录数
search.forEach(hit -> {
AuditEntity content = hit.getContent();
auditEntityPage.forEach(audit -> {
AuditListResult result = new AuditListResult();
result.setId(content.getId().toString());
result.setEventStatus(content.getEventStatus());
result.setEventType(content.getEventType().getDesc());
result.setEventTime(content.getEventTime());
result.setId(audit.getId().toString());
result.setEventStatus(audit.getEventStatus());
result.setEventType(audit.getEventType().getDesc());
result.setEventTime(audit.getEventTime());
//用户代理
result.setUserAgent(content.getUserAgent());
result.setGeoLocation(content.getGeoLocation());
result.setUserAgent(audit.getUserAgent());
result.setGeoLocation(audit.getGeoLocation());
//用户ID
result.setUserId(content.getActorId());
result.setUsername(getUsername(content.getActorType(), content.getActorId()));
result.setUserId(audit.getActorId());
result.setUsername(getUsername(audit.getActorType(), audit.getActorId()));
//用户类型
result.setUserType(content.getActorType().getType());
result.setUserType(audit.getActorType().getType());
//操作对象
if (Objects.nonNull(content.getTargets())) {
for (Target target : content.getTargets()) {
if (Objects.nonNull(audit.getTargets())) {
for (Target target : audit.getTargets()) {
if (Objects.nonNull(target.getId())) {
target.setName(getTargetName(target.getType(), target.getId()));
}
target.setTypeName(target.getType().getDesc());
}
}
result.setTargets(content.getTargets());
result.setTargets(audit.getTargets());
list.add(result);
});
//@formatter:off
Page<AuditListResult> result = new Page<>();
result.setPagination(Page.Pagination.builder()
.total(search.getTotalHits())
.totalPages(Math.toIntExact(search.getTotalHits() / page.getPageSize()))
.total(auditEntityPage.getTotalElements())
.totalPages(auditEntityPage.getTotalPages())
.current(page.getCurrent() + 1)
.build());
result.setList(list);
@ -165,13 +145,12 @@ public interface AuditDataConverter {
* 审计列表请求到本机搜索查询
*
* @param query {@link AuditListQuery}
* @param page {@link PageModel}
* @return {@link NativeQuery}
* @return {@link Predicate}
*/
default NativeQuery auditListRequestConvertToNativeQuery(AuditListQuery query, PageModel page) {
//构建查询 builder下有 must、should 以及 mustNot 相当于 sql 中的 and、or 以及 not
BoolQuery.Builder queryBuilder = QueryBuilders.bool();
List<SortOptions> fieldSortBuilders = Lists.newArrayList();
default Predicate auditListRequestConvertToPredicate(AuditListQuery query) {
QAuditEntity auditEntity = QAuditEntity.auditEntity;
Predicate predicate = ExpressionUtils.and(auditEntity.isNotNull(),
auditEntity.deleted.eq(Boolean.FALSE));
//用户名存在，查询用户ID
if (StringUtils.hasText(query.getUsername())) {
String actorId = "";
@ -182,6 +161,8 @@ public interface AuditDataConverter {
if (!Objects.isNull(user)) {
actorId = user.getId().toString();
}
// 用户类型
predicate = ExpressionUtils.and(predicate, auditEntity.actorType.eq(UserType.USER));
}
if (UserType.ADMIN.getType().equals(query.getUserType())) {
AdministratorRepository administratorRepository = ApplicationContextHelp
@ -191,61 +172,45 @@ public interface AuditDataConverter {
if (optional.isPresent()) {
actorId = optional.get().getId().toString();
}
// 用户类型
predicate = ExpressionUtils.and(predicate,
auditEntity.actorType.eq(UserType.ADMIN));
}
queryBuilder.must(Queries.termQueryAsQuery(ACTOR_ID, actorId));
predicate = ExpressionUtils.and(predicate, auditEntity.actorId.eq(actorId));
}
//用户类型
queryBuilder.must(Queries.termQueryAsQuery(ACTOR_TYPE, query.getUserType()));
//事件类型
if (!CollectionUtils.isEmpty(query.getEventType())) {
queryBuilder.must(QueryBuilders.terms(builder -> {
builder
.terms(
new TermsQueryField.Builder()
.value(query.getEventType().stream()
.map(t -> FieldValue.of(t.getCode())).collect(Collectors.toList()))
.build());
builder.field(EVENT_TYPE);
return builder;
}));
predicate = ExpressionUtils.and(predicate,
auditEntity.eventType.in(query.getEventType()));
}
//事件状态
if (Objects.nonNull(query.getEventStatus())) {
queryBuilder
.must(Queries.termQueryAsQuery(EVENT_STATUS, query.getEventStatus().getCode()));
predicate = ExpressionUtils.and(predicate,
auditEntity.eventStatus.in(query.getEventStatus()));
}
//字段排序
page.getSorts().forEach(sort -> {
SortOrder sortOrder;
if (org.apache.commons.lang3.StringUtils.equals(sort.getSorter(), SORT_EVENT_TIME)) {
if (sort.getAsc()) {
sortOrder = SortOrder.Asc;
} else {
sortOrder = SortOrder.Desc;
}
} else {
sortOrder = SortOrder.Desc;
}
SortOptions eventTimeSortBuilder = SortOptions
.of(s -> s.field(FieldSort.of(f -> f.field(EVENT_TIME).order(sortOrder))));
fieldSortBuilders.add(eventTimeSortBuilder);
});
// page.getSorts().forEach(sort -> {
// SortOrder sortOrder;
// if (org.apache.commons.lang3.StringUtils.equals(sort.getSorter(), SORT_EVENT_TIME)) {
// if (sort.getAsc()) {
// sortOrder = SortOrder.Asc;
// } else {
// sortOrder = SortOrder.Desc;
// }
// } else {
// sortOrder = SortOrder.Desc;
// }
// SortOptions eventTimeSortBuilder = SortOptions
// .of(s -> s.field(FieldSort.of(f -> f.field(EVENT_TIME).order(sortOrder))));
// fieldSortBuilders.add(eventTimeSortBuilder);
// });
//事件时间
if (!Objects.isNull(query.getStartEventTime())
&& !Objects.isNull(query.getEndEventTime())) {
queryBuilder.must(QueryBuilders.range(r -> r.field(EVENT_TIME)
.gte(JsonData.of(query.getStartEventTime()
.format(DateTimeFormatter.ofPattern(DEFAULT_DATE_TIME_FORMATTER_PATTERN))))
.lte(JsonData.of(query.getEndEventTime()
.format(DateTimeFormatter.ofPattern(DEFAULT_DATE_TIME_FORMATTER_PATTERN))))
.timeZone(ZoneId.systemDefault().getId())
.format(DEFAULT_DATE_TIME_FORMATTER_PATTERN)));
predicate = ExpressionUtils.and(predicate,
auditEntity.eventTime.between(query.getStartEventTime(), query.getEndEventTime()));
}
return new NativeQueryBuilder().withQuery(queryBuilder.build()._toQuery())
//分页参数
.withPageable(PageRequest.of(page.getCurrent(), page.getPageSize()))
//排序
.withSort(fieldSortBuilders).build();
return predicate;
}
/**

62
eiam-audit/src/main/java/cn/topiam/employee/audit/service/impl/AuditServiceImpl.java
View File

@ -17,33 +17,36 @@
*/
package cn.topiam.employee.audit.service.impl;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.client.elc.NativeQuery;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.querydsl.QPageRequest;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import com.querydsl.core.types.OrderSpecifier;
import com.querydsl.core.types.Predicate;
import cn.topiam.employee.audit.controller.pojo.AuditListQuery;
import cn.topiam.employee.audit.controller.pojo.AuditListResult;
import cn.topiam.employee.audit.controller.pojo.DictResult;
import cn.topiam.employee.audit.entity.AuditEntity;
import cn.topiam.employee.audit.entity.QAuditEntity;
import cn.topiam.employee.audit.event.type.EventType;
import cn.topiam.employee.audit.repository.AuditRepository;
import cn.topiam.employee.audit.service.AuditService;
import cn.topiam.employee.audit.service.converter.AuditDataConverter;
import cn.topiam.employee.support.autoconfiguration.SupportProperties;
import cn.topiam.employee.support.exception.BadParamsException;
import cn.topiam.employee.support.repository.page.domain.Page;
import cn.topiam.employee.support.repository.page.domain.PageModel;
import cn.topiam.employee.support.security.userdetails.UserType;
import cn.topiam.employee.support.security.util.SecurityUtils;
import static cn.topiam.employee.common.constant.AuditConstants.getAuditIndexPrefix;
import lombok.RequiredArgsConstructor;
import static cn.topiam.employee.audit.service.converter.AuditDataConverter.SORT_EVENT_TIME;
import static cn.topiam.employee.support.security.userdetails.UserType.USER;
/**
@ -53,6 +56,7 @@ import static cn.topiam.employee.support.security.userdetails.UserType.USER;
* Created by support@topiam.cn on 2021/9/10 23:06
*/
@Service
@RequiredArgsConstructor
public class AuditServiceImpl implements AuditService {
/**
@ -69,13 +73,21 @@ public class AuditServiceImpl implements AuditService {
throw new BadParamsException("用户类型错误");
}
//查询入参转查询条件
NativeQuery nsq = auditDataConverter.auditListRequestConvertToNativeQuery(query, page);
Predicate predicate = auditDataConverter.auditListRequestConvertToPredicate(query);
// 字段排序
OrderSpecifier<LocalDateTime> order = QAuditEntity.auditEntity.eventTime.desc();
for (PageModel.Sort sort : page.getSorts()) {
if (org.apache.commons.lang3.StringUtils.equals(sort.getSorter(), SORT_EVENT_TIME)) {
if (sort.getAsc()) {
order = QAuditEntity.auditEntity.eventTime.asc();
}
}
}
//分页条件
QPageRequest request = QPageRequest.of(page.getCurrent(), page.getPageSize(), order);
//查询列表
SearchHits<AuditEntity> search = elasticsearchTemplate.search(nsq, AuditEntity.class,
IndexCoordinates
.of(getAuditIndexPrefix(supportProperties.getAudit().getIndexPrefix()) + "*"));
//结果转返回结果
return auditDataConverter.searchHitsConvertToAuditListResult(search, page);
return auditDataConverter
.entityConvertToAuditListResult(auditRepository.findAll(predicate, request), page);
}
/**
@ -116,27 +128,13 @@ public class AuditServiceImpl implements AuditService {
return list;
}
/**
* AuditProperties
*/
private final SupportProperties supportProperties;
/**
* ElasticsearchTemplate
*/
private final ElasticsearchTemplate elasticsearchTemplate;
/**
* AuditDataConverter
*/
private final AuditDataConverter auditDataConverter;
public AuditServiceImpl(SupportProperties supportProperties,
ElasticsearchTemplate elasticsearchTemplate,
AuditDataConverter auditDataConverter) {
this.supportProperties = supportProperties;
this.elasticsearchTemplate = elasticsearchTemplate;
this.auditDataConverter = auditDataConverter;
}
private final AuditDataConverter auditDataConverter;
/**
* AuditRepository
*/
private final AuditRepository auditRepository;
}

8
eiam-common/src/main/java/cn/topiam/employee/common/repository/account/impl/UserRepositoryCustomizedImpl.java
View File

@ -31,7 +31,6 @@ import org.springframework.cache.annotation.CacheEvict;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
@ -352,10 +351,5 @@ public class UserRepositoryCustomizedImpl implements UserRepositoryCustomized {
/**
* JdbcTemplate
*/
private final JdbcTemplate jdbcTemplate;
/**
* ElasticsearchTemplate
*/
private final ElasticsearchTemplate elasticsearchTemplate;
private final JdbcTemplate jdbcTemplate;
}

2
eiam-common/src/main/java/cn/topiam/employee/common/repository/app/AppRepository.java
View File

@ -105,7 +105,7 @@ public interface AppRepository extends LogicDeleteRepository<AppEntity, Long>,
*/
@NotNull
@Cacheable
@Query(value = "SELECT AppEntity FROM AppEntity WHERE id = :id")
@Query(value = "FROM AppEntity WHERE id = :id")
Optional<AppEntity> findByIdContainsDeleted(@NotNull @Param(value = "id") Long id);
/**

113
eiam-console/src/main/java/cn/topiam/employee/console/converter/account/UserConverter.java
View File

@ -23,19 +23,15 @@ import java.util.List;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.client.elc.NativeQuery;
import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
import org.springframework.data.elasticsearch.client.elc.Queries;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import com.google.common.collect.Lists;
import com.querydsl.core.types.ExpressionUtils;
import com.querydsl.core.types.Predicate;
import cn.topiam.employee.audit.entity.AuditElasticSearchEntity;
import cn.topiam.employee.audit.entity.Event;
import cn.topiam.employee.audit.event.type.EventType;
import cn.topiam.employee.audit.entity.AuditEntity;
import cn.topiam.employee.audit.entity.QAuditEntity;
import cn.topiam.employee.audit.event.type.PortalEventType;
import cn.topiam.employee.common.constant.CommonConstants;
import cn.topiam.employee.common.entity.account.UserDetailEntity;
@ -52,20 +48,9 @@ import cn.topiam.employee.console.pojo.update.account.UserUpdateParam;
import cn.topiam.employee.support.context.ApplicationContextHelp;
import cn.topiam.employee.support.repository.page.domain.Page;
import cn.topiam.employee.support.repository.page.domain.PageModel;
import co.elastic.clients.elasticsearch._types.FieldSort;
import co.elastic.clients.elasticsearch._types.FieldValue;
import co.elastic.clients.elasticsearch._types.SortOptions;
import co.elastic.clients.elasticsearch._types.SortOrder;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import co.elastic.clients.elasticsearch._types.query_dsl.TermsQueryField;
import static cn.topiam.employee.audit.entity.Actor.ACTOR_ID;
import static cn.topiam.employee.audit.entity.Event.EVENT_TIME;
import static cn.topiam.employee.audit.entity.Event.EVENT_TYPE;
import static cn.topiam.employee.audit.enums.TargetType.PORTAL;
import static cn.topiam.employee.audit.event.type.EventType.APP_SSO;
import static cn.topiam.employee.audit.event.type.EventType.LOGIN_PORTAL;
import static cn.topiam.employee.audit.service.converter.AuditDataConverter.SORT_EVENT_TIME;
import static cn.topiam.employee.common.util.ImageAvatarUtils.bufferedImageToBase64;
import static cn.topiam.employee.common.util.ImageAvatarUtils.generateAvatarImg;
import static cn.topiam.employee.support.util.PhoneNumberUtils.getPhoneAreaCode;
@ -259,90 +244,52 @@ public interface UserConverter {
* 审计列表请求到本机搜索查询
*
* @param id {@link Long}
* @param page {@link PageModel}
* @return {@link NativeQuery}
*/
default NativeQuery auditListRequestConvertToNativeQuery(Long id, PageModel page) {
//构建查询 builder下有 must、should 以及 mustNot 相当于 sql 中的 and、or 以及 not
BoolQuery.Builder queryBuilder = QueryBuilders.bool();
List<SortOptions> fieldSortBuilders = Lists.newArrayList();
//事件类型
List<FieldValue> set = new ArrayList<>();
set.add(FieldValue.of(LOGIN_PORTAL.getCode()));
set.add(FieldValue.of(EventType.APP_SSO.getCode()));
queryBuilder.must(QueryBuilders.terms(builder -> {
builder.terms(new TermsQueryField.Builder().value(set).build());
builder.field(EVENT_TYPE);
return builder;
}));
//用户id
queryBuilder.must(Queries.termQueryAsQuery(ACTOR_ID, id.toString()));
//字段排序
page.getSorts().forEach(sort -> {
co.elastic.clients.elasticsearch._types.SortOrder sortOrder;
if (org.apache.commons.lang3.StringUtils.equals(sort.getSorter(), SORT_EVENT_TIME)) {
if (sort.getAsc()) {
sortOrder = co.elastic.clients.elasticsearch._types.SortOrder.Asc;
} else {
sortOrder = SortOrder.Desc;
}
} else {
sortOrder = SortOrder.Desc;
}
SortOptions eventTimeSortBuilder = SortOptions
.of(s -> s.field(FieldSort.of(f -> f.field(EVENT_TIME).order(sortOrder))));
fieldSortBuilders.add(eventTimeSortBuilder);
});
NativeQueryBuilder nativeQueryBuilder = new NativeQueryBuilder()
.withQuery(queryBuilder.build()._toQuery())
//分页参数
.withPageable(PageRequest.of(page.getCurrent(), page.getPageSize()));
if (!CollectionUtils.isEmpty(fieldSortBuilders)) {
//排序
nativeQueryBuilder.withSort(fieldSortBuilders);
}
return nativeQueryBuilder.build();
default Predicate auditListRequestConvertToNativeQuery(Long id) {
QAuditEntity auditEntity = QAuditEntity.auditEntity;
return ExpressionUtils.and(auditEntity.isNotNull(),
auditEntity.deleted.eq(Boolean.FALSE).and(auditEntity.actorId.eq(id.toString()))
.and(auditEntity.eventType.in(LOGIN_PORTAL, APP_SSO)));
}
/**
* searchHits 转用户登录日志列表
*
* @param search {@link SearchHits}
* @param auditEntityPage {@link Page}
* @param page {@link PageModel}
* @return {@link Page}
*/
default Page<UserLoginAuditListResult> searchHitsConvertToAuditListResult(SearchHits<AuditElasticSearchEntity> search,
PageModel page) {
default Page<UserLoginAuditListResult> entityConvertToAuditListResult(org.springframework.data.domain.Page<AuditEntity> auditEntityPage,
PageModel page) {
List<UserLoginAuditListResult> list = new ArrayList<>();
//总记录数
search.forEach(hit -> {
AuditElasticSearchEntity content = hit.getContent();
Event event = content.getEvent();
auditEntityPage.forEach(audit -> {
UserLoginAuditListResult result = new UserLoginAuditListResult();
//单点登录
if (event.getType().getCode().equals(PortalEventType.APP_SSO.getCode())) {
result.setAppName(getAppName(content.getTargets().get(0).getId()));
if (audit.getEventType().getCode().equals(PortalEventType.APP_SSO.getCode())) {
result.setAppName(getAppName(audit.getTargets().get(0).getId()));
}
//登录门户
if (event.getType().getCode().equals(PortalEventType.LOGIN_PORTAL.getCode())) {
if (audit.getEventType().getCode().equals(PortalEventType.LOGIN_PORTAL.getCode())) {
result.setAppName(PORTAL.getDesc());
}
result.setEventTime(event.getTime());
result.setClientIp(content.getGeoLocation().getIp());
result.setBrowser(content.getUserAgent().getBrowser());
result.setLocation(content.getGeoLocation().getCityName());
result.setEventStatus(event.getStatus());
result.setEventTime(audit.getEventTime());
result.setClientIp(audit.getGeoLocation().getIp());
result.setBrowser(audit.getUserAgent().getBrowser());
result.setLocation(audit.getGeoLocation().getCityName());
result.setEventStatus(audit.getEventStatus());
list.add(result);
});
//@formatter:off
Page<UserLoginAuditListResult> result = new Page<>();
result.setPagination(Page.Pagination.builder()
.total(search.getTotalHits())
.totalPages(Math.toIntExact(search.getTotalHits() / page.getPageSize()))
.current(page.getCurrent() + 1)
.build());
result.setList(list);
//@formatter:on
Page<UserLoginAuditListResult> result = new Page<>();
result.setPagination(Page.Pagination.builder()
.total(auditEntityPage.getTotalElements())
.totalPages(auditEntityPage.getTotalPages())
.current(page.getCurrent() + 1)
.build());
result.setList(list);
//@formatter:on
return result;
}

49
eiam-console/src/main/java/cn/topiam/employee/console/service/account/impl/UserServiceImpl.java
View File

@ -25,10 +25,7 @@ import java.util.*;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.client.elc.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.client.elc.NativeQuery;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.querydsl.QPageRequest;
import org.springframework.http.HttpStatus;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
@ -37,12 +34,15 @@ import org.springframework.transaction.annotation.Transactional;
import com.google.i18n.phonenumbers.NumberParseException;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.google.i18n.phonenumbers.Phonenumber;
import com.querydsl.core.types.OrderSpecifier;
import com.querydsl.core.types.Predicate;
import com.querydsl.core.types.dsl.BooleanExpression;
import cn.topiam.employee.audit.context.AuditContext;
import cn.topiam.employee.audit.entity.AuditElasticSearchEntity;
import cn.topiam.employee.audit.entity.QAuditEntity;
import cn.topiam.employee.audit.entity.Target;
import cn.topiam.employee.audit.enums.TargetType;
import cn.topiam.employee.audit.repository.AuditRepository;
import cn.topiam.employee.common.entity.account.*;
import cn.topiam.employee.common.entity.account.po.UserPO;
import cn.topiam.employee.common.entity.account.query.UserListNotInGroupQuery;
@ -61,7 +61,6 @@ import cn.topiam.employee.console.service.account.UserService;
import cn.topiam.employee.core.message.MsgVariable;
import cn.topiam.employee.core.message.mail.MailMsgEventPublish;
import cn.topiam.employee.core.message.sms.SmsMsgEventPublish;
import cn.topiam.employee.support.autoconfiguration.SupportProperties;
import cn.topiam.employee.support.exception.BadParamsException;
import cn.topiam.employee.support.exception.InfoValidityFailException;
import cn.topiam.employee.support.exception.TopIamException;
@ -76,7 +75,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import static cn.topiam.employee.audit.enums.TargetType.USER;
import static cn.topiam.employee.audit.enums.TargetType.USER_DETAIL;
import static cn.topiam.employee.common.constant.AuditConstants.getAuditIndexPrefix;
import static cn.topiam.employee.audit.service.converter.AuditDataConverter.SORT_EVENT_TIME;
import static cn.topiam.employee.core.message.sms.SmsMsgEventPublish.USERNAME;
import static cn.topiam.employee.support.repository.domain.BaseEntity.LAST_MODIFIED_BY;
import static cn.topiam.employee.support.repository.domain.BaseEntity.LAST_MODIFIED_TIME;
@ -501,13 +500,21 @@ public class UserServiceImpl implements UserService {
@Override
public Page<UserLoginAuditListResult> findUserLoginAuditList(Long id, PageModel pageModel) {
//查询入参转查询条件
NativeQuery nsq = userConverter.auditListRequestConvertToNativeQuery(id, pageModel);
//查询列表
SearchHits<AuditElasticSearchEntity> search = elasticsearchTemplate.search(nsq,
AuditElasticSearchEntity.class, IndexCoordinates
.of(getAuditIndexPrefix(supportProperties.getAudit().getIndexPrefix() + "*")));
//结果转返回结果
return userConverter.searchHitsConvertToAuditListResult(search, pageModel);
Predicate predicate = userConverter.auditListRequestConvertToNativeQuery(id);
// 字段排序
OrderSpecifier<LocalDateTime> order = QAuditEntity.auditEntity.eventTime.desc();
for (PageModel.Sort sort : pageModel.getSorts()) {
if (org.apache.commons.lang3.StringUtils.equals(sort.getSorter(), SORT_EVENT_TIME)) {
if (sort.getAsc()) {
order = QAuditEntity.auditEntity.eventTime.asc();
}
}
}
//分页条件
QPageRequest request = QPageRequest.of(pageModel.getCurrent(), pageModel.getPageSize(),
order);
return userConverter
.entityConvertToAuditListResult(auditRepository.findAll(predicate, request), pageModel);
}
/**
@ -572,11 +579,6 @@ public class UserServiceImpl implements UserService {
*/
private final UserHistoryPasswordRepository userHistoryPasswordRepository;
/**
* ElasticsearchTemplate
*/
private final ElasticsearchTemplate elasticsearchTemplate;
/**
* 邮件消息发布
*/
@ -587,14 +589,13 @@ public class UserServiceImpl implements UserService {
*/
private final SmsMsgEventPublish smsMsgEventPublish;
/**
* EiamSupportProperties
*/
private final SupportProperties supportProperties;
/**
* PasswordPolicyManager
*/
private final PasswordPolicyManager<UserEntity> passwordPolicyManager;
/**
* AuditRepository
*/
private final AuditRepository auditRepository;
}

88
eiam-console/src/main/java/cn/topiam/employee/console/service/analysis/impl/AnalysisServiceImpl.java
View File

@ -18,16 +18,11 @@
package cn.topiam.employee.console.service.analysis.impl;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.util.*;
import org.jetbrains.annotations.NotNull;
import org.springframework.data.elasticsearch.client.elc.*;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import cn.topiam.employee.audit.entity.AuditElasticSearchEntity;
import cn.topiam.employee.audit.event.type.EventType;
import cn.topiam.employee.audit.repository.AuditRepository;
import cn.topiam.employee.audit.repository.result.AuditStatisticsResult;
@ -40,19 +35,10 @@ import cn.topiam.employee.common.repository.authentication.IdentityProviderRepos
import cn.topiam.employee.console.pojo.query.analysis.AnalysisQuery;
import cn.topiam.employee.console.pojo.result.analysis.*;
import cn.topiam.employee.console.service.analysis.AnalysisService;
import cn.topiam.employee.support.autoconfiguration.SupportProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
import co.elastic.clients.json.JsonData;
import static cn.topiam.employee.audit.entity.Event.*;
import static cn.topiam.employee.console.converter.authn.IdentityProviderConverter.getIdentityProviderType;
import static cn.topiam.employee.support.constant.EiamConstants.DEFAULT_DATE_TIME_FORMATTER_PATTERN;
/**
* @author TopIAM
@ -63,8 +49,6 @@ import static cn.topiam.employee.support.constant.EiamConstants.DEFAULT_DATE_TIM
@RequiredArgsConstructor
public class AnalysisServiceImpl implements AnalysisService {
public static final String COUNT = "count";
/**
* 概述
*
@ -116,22 +100,6 @@ public class AnalysisServiceImpl implements AnalysisService {
return applicationVisitList;
}
/**
* 时间查询条件
*
* @param params {@link AnalysisQuery}
* @return {@link Query}
*/
private Query getRangeQueryBuilder(AnalysisQuery params) {
String min = params.getStartTime()
.format(DateTimeFormatter.ofPattern(DEFAULT_DATE_TIME_FORMATTER_PATTERN));
String max = params.getEndTime()
.format(DateTimeFormatter.ofPattern(DEFAULT_DATE_TIME_FORMATTER_PATTERN));
// 查询条件
return QueryBuilders.range(range -> range.field(EVENT_TIME).timeZone(ZONE_ID)
.format(DEFAULT_DATE_TIME_FORMATTER_PATTERN).gt(JsonData.of(min)).lt(JsonData.of(max)));
}
/**
* 热门认证方式
* @param params {@link AnalysisQuery}
@ -156,7 +124,7 @@ public class AnalysisServiceImpl implements AnalysisService {
* 登录区域统计
*
* @param params {@link AnalysisQuery}
* @return {@link List< AuthnZoneResult >}
* @return {@link List<AuthnZoneResult>}
*/
@Override
public List<AuthnZoneResult> authnZone(AnalysisQuery params) {
@ -182,60 +150,6 @@ public class AnalysisServiceImpl implements AnalysisService {
return app.getName();
}
/**
* ES聚合查询
*
* @param searchHits {@link SearchHits<AuditElasticSearchEntity>}
* @return {@link Aggregation}
*/
private ElasticsearchAggregation getCountAggregation(SearchHits<AuditElasticSearchEntity> searchHits) {
ElasticsearchAggregations elasticsearchAggregations = (ElasticsearchAggregations) searchHits
.getAggregations();
if (elasticsearchAggregations == null) {
return null;
}
List<ElasticsearchAggregation> aggregations = elasticsearchAggregations.aggregations();
return aggregations.stream()
.filter(aggregation -> aggregation.aggregation().getName().equals(COUNT)).findFirst()
.orElse(null);
}
/**
* 拼装查询条件
*
* @param query {@link Query}
* @param eventType {@link EventType}
* @return {@link BoolQuery.Builder}
*/
@NotNull
private BoolQuery.Builder getQueryBuilder(Query query, EventType eventType) {
// 查询条件
BoolQuery.Builder queryBuilder = QueryBuilders.bool();
// 事件类型
queryBuilder.must(Queries.termQueryAsQuery(EVENT_TYPE, eventType.getCode()));
// 日期条件
queryBuilder.filter(query);
return queryBuilder;
}
/**
* 拼装查询条件
*
* @param query {@link Query}
* @param eventType {@link EventType}
* @return {@link Query}
*/
@NotNull
private Query getQuery(Query query, EventType eventType) {
return getQueryBuilder(query, eventType).build()._toQuery();
}
private final String ZONE_ID = ZoneId.systemDefault().getId();
private final SupportProperties supportProperties;
// private final ElasticsearchTemplate elasticsearchTemplate;
private final AuditRepository auditRepository;
private final AppRepository appRepository;