Implemented paged operations and used for database cleanup tasks.

pull/1187/head
strangeweaver 2016-02-18 12:39:05 +00:00 committed by Justin Richer
parent 099211593c
commit 46046b574a
13 changed files with 534 additions and 58 deletions

View File

@ -0,0 +1,160 @@
package org.mitre.data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
/**
* Abstract class for performing an operation on a potentially large
* number of items by paging through the items in discreet chunks.
*
* @param <T> the type parameter
* @author Colm Smyth.
*/
public abstract class AbstractPageOperationTemplate<T> {
private static final Logger logger = LoggerFactory.getLogger(AbstractPageOperationTemplate.class);
private static int DEFAULT_MAX_PAGES = 1000;
private static long DEFAULT_MAX_TIME_MILLIS = 600000L; //10 Minutes
/**
* int specifying the maximum number of
* pages which should be fetched before
* execution should terminate
*/
private int maxPages;
/**
* long specifying the maximum execution time
* in milliseconds
*/
private long maxTime;
/**
* boolean specifying whether or not Exceptions
* incurred performing the operation should be
* swallowed during execution default true.
*/
private boolean swallowExceptions = true;
/**
* default constructor which sets the value of
* maxPages and maxTime to DEFAULT_MAX_PAGES and
* DEFAULT_MAX_TIME_MILLIS respectively
*/
public AbstractPageOperationTemplate(){
this(DEFAULT_MAX_PAGES, DEFAULT_MAX_TIME_MILLIS);
}
/**
* Instantiates a new AbstractPageOperationTemplate with the
* given maxPages and maxTime
*
* @param maxPages the maximum number of pages to fetch.
* @param maxTime the maximum execution time.
*/
public AbstractPageOperationTemplate(int maxPages, long maxTime){
this.maxPages = maxPages;
this.maxTime = maxTime;
}
/**
* Execute the operation on each member of a page of results
* retrieved through the fetch method. the method will execute
* until either the maxPages or maxTime limit is reached or until
* the fetch method returns no more results. Exceptions thrown
* performing the operation on the item will be swallowed if the
* swallowException (default true) field is set true.
*/
public void execute(){
logger.info("Starting execution of paged operation. maximum time: " + maxTime
+ " maximum pages: " + maxPages);
long startTime = System.currentTimeMillis();
long executionTime = 0;
int i = 0;
int exceptionsSwallowedCount = 0;
int operationsCompleted = 0;
Set<String> exceptionsSwallowedClasses = new HashSet<String>();
while (i< maxPages && executionTime < maxTime){
Collection<T> page = fetchPage();
if(page == null || page.size() == 0){
break;
}
for (T item : page) {
try {
doOperation(item);
operationsCompleted++;
} catch (Exception e){
if(swallowExceptions){
exceptionsSwallowedCount++;
exceptionsSwallowedClasses.add(e.getClass().getName());
logger.debug("Swallowing exception " + e.getMessage(), e);
} else {
logger.debug("Rethrowing exception " + e.getMessage());
throw e;
}
}
}
i++;
executionTime = System.currentTimeMillis() - startTime;
}
logger.info("Paged operation run completed " + operationsCompleted + " swallowed " + exceptionsSwallowedCount + " exceptions");
for(String className: exceptionsSwallowedClasses) {
logger.warn("Paged operation swallowed at least one exception of type " + className);
}
}
/**
* method responsible for fetching
* a page of items.
*
* @return the collection of items
*/
public abstract Collection<T> fetchPage();
/**
* method responsible for performing desired
* operation on a fetched page item.
*
* @param item the item
*/
protected abstract void doOperation(T item);
public int getMaxPages() {
return maxPages;
}
public void setMaxPages(int maxPages) {
this.maxPages = maxPages;
}
public long getMaxTime() {
return maxTime;
}
public void setMaxTime(long maxTime) {
this.maxTime = maxTime;
}
public boolean isSwallowExceptions() {
return swallowExceptions;
}
public void setSwallowExceptions(boolean swallowExceptions) {
this.swallowExceptions = swallowExceptions;
}
}

View File

@ -0,0 +1,35 @@
package org.mitre.data;
/**
* Default implementation of PageCriteria which specifies
* both page to be retrieved and page size in the constructor.
*
* @author Colm Smyth
*/
public class DefaultPageCriteria implements PageCriteria {
private static final int DEFAULT_PAGE_NUMBER = 0;
private static final int DEFAULT_PAGE_SIZE = 100;
private int pageNumber;
private int pageSize;
public DefaultPageCriteria(){
this(DEFAULT_PAGE_NUMBER, DEFAULT_PAGE_SIZE);
}
public DefaultPageCriteria(int pageNumber, int pageSize) {
this.pageNumber = pageNumber;
this.pageSize = pageSize;
}
@Override
public int getPageNumber() {
return pageNumber;
}
@Override
public int getPageSize() {
return pageSize;
}
}

View File

@ -0,0 +1,13 @@
package org.mitre.data;
/**
* Interface which defines page criteria for use in
* a repository operation.
*
* @author Colm Smyth
*/
public interface PageCriteria {
public int getPageNumber();
public int getPageSize();
}

View File

@ -18,6 +18,7 @@ package org.mitre.oauth2.repository;
import java.util.List;
import org.mitre.data.PageCriteria;
import org.mitre.oauth2.model.AuthenticationHolderEntity;
public interface AuthenticationHolderRepository {
@ -31,5 +32,5 @@ public interface AuthenticationHolderRepository {
public List<AuthenticationHolderEntity> getOrphanedAuthenticationHolders();
public List<AuthenticationHolderEntity> getOrphanedAuthenticationHolders(PageCriteria pageCriteria);
}

View File

@ -18,6 +18,7 @@ package org.mitre.oauth2.repository;
import java.util.Collection;
import org.mitre.data.PageCriteria;
import org.mitre.oauth2.model.AuthorizationCodeEntity;
/**
@ -56,4 +57,10 @@ public interface AuthorizationCodeRepository {
*/
public Collection<AuthorizationCodeEntity> getExpiredCodes();
/**
* @return A collection of all expired codes, limited by the given
* PageCriteria.
*/
public Collection<AuthorizationCodeEntity> getExpiredCodes(PageCriteria pageCriteria);
}

View File

@ -19,6 +19,7 @@ package org.mitre.oauth2.repository;
import java.util.List;
import java.util.Set;
import org.mitre.data.PageCriteria;
import org.mitre.oauth2.model.ClientDetailsEntity;
import org.mitre.oauth2.model.OAuth2AccessTokenEntity;
import org.mitre.oauth2.model.OAuth2RefreshTokenEntity;
@ -57,8 +58,12 @@ public interface OAuth2TokenRepository {
public Set<OAuth2AccessTokenEntity> getAllExpiredAccessTokens();
public Set<OAuth2AccessTokenEntity> getAllExpiredAccessTokens(PageCriteria pageCriteria);
public Set<OAuth2RefreshTokenEntity> getAllExpiredRefreshTokens();
public Set<OAuth2RefreshTokenEntity> getAllExpiredRefreshTokens(PageCriteria pageCriteria);
public Set<OAuth2AccessTokenEntity> getAccessTokensForResourceSet(ResourceSet rs);
/**

View File

@ -16,9 +16,12 @@
*******************************************************************************/
package org.mitre.util.jpa;
import org.mitre.data.PageCriteria;
import java.util.List;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
/**
* @author mfranklin
@ -26,7 +29,7 @@ import javax.persistence.EntityManager;
* Time: 2:13 PM
*/
public class JpaUtil {
public static <T> T getSingleResult(List<T> list) {
public static <T> T getSingleResult(List<T> list) {
switch(list.size()) {
case 0:
return null;
@ -37,6 +40,25 @@ public class JpaUtil {
}
}
/**
* Get a page of results from the specified TypedQuery
* by using the given PageCriteria to limit the query
* results. The PageCriteria will override any size or
* offset already specified on the query.
*
* @param <T> the type parameter
* @param query the query
* @param pageCriteria the page criteria
* @return the list
*/
public static <T> List<T> getResultPage(TypedQuery<T> query, PageCriteria pageCriteria){
query.setMaxResults(pageCriteria.getPageSize());
query.setFirstResult(pageCriteria.getPageNumber()*pageCriteria.getPageSize());
return query.getResultList();
}
public static <T, I> T saveOrUpdate(I id, EntityManager entityManager, T entity) {
T tmp = entityManager.merge(entity);
entityManager.flush();

View File

@ -0,0 +1,204 @@
package org.mitre.data;
import org.junit.Before;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* @author Colm Smyth
*/
public class AbstractPageOperationTemplateTest {
@Before
public void setUp() throws Exception {
}
@Test
public void execute_zeropages() {
CountingPageOperation op = new CountingPageOperation(0,Long.MAX_VALUE);
op.execute();
assertEquals(0L, op.counter);
}
@Test
public void execute_singlepage() {
CountingPageOperation op = new CountingPageOperation(1,Long.MAX_VALUE);
op.execute();
assertEquals(10L, op.counter);
}
@Test
public void execute_negpage() {
CountingPageOperation op = new CountingPageOperation(-1,Long.MAX_VALUE);
op.execute();
assertEquals(0L, op.counter);
}
@Test
public void execute_npage(){
int n = 7;
CountingPageOperation op = new CountingPageOperation(n,Long.MAX_VALUE);
op.execute();
assertEquals(n*10L, op.counter);
}
@Test
public void execute_nullpage(){
CountingPageOperation op = new NullPageCountingPageOperation(Integer.MAX_VALUE, Long.MAX_VALUE);
op.execute();
assertEquals(0L, op.getCounter());
}
@Test
public void execute_emptypage(){
CountingPageOperation op = new EmptyPageCountingPageOperation(Integer.MAX_VALUE, Long.MAX_VALUE);
op.execute();
assertEquals(0L, op.getCounter());
}
@Test
public void execute_zerotime(){
CountingPageOperation op = new CountingPageOperation(Integer.MAX_VALUE,0L);
op.execute();
assertEquals(0L, op.getCounter());
assertEquals(0L, op.getLastFetchTime());
}
@Test
public void execute_nonzerotime(){
Long timeMillis = 100L;
CountingPageOperation op = new CountingPageOperation(Integer.MAX_VALUE,timeMillis);
op.execute();
assertTrue("start time " + op.getStartTime() + "" +
" to last fetch time " + op.getLastFetchTime() +
" exceeds max time" + timeMillis, op.getLastFetchTime() - op.getStartTime() <= timeMillis);
}
@Test
public void execute_negtime(){
Long timeMillis = -100L;
CountingPageOperation op = new CountingPageOperation(Integer.MAX_VALUE,timeMillis);
op.execute();
assertEquals(0L, op.getCounter());
}
@Test
public void execute_swallowException(){
CountingPageOperation op = new EvenExceptionCountingPageOperation(1, 1000L);
op.execute();
assertTrue(op.isSwallowExceptions());
assertEquals(5L, op.getCounter());
}
@Test(expected = IllegalStateException.class)
public void execute_noSwallowException(){
CountingPageOperation op = new EvenExceptionCountingPageOperation(1, 1000L);
op.setSwallowExceptions(false);
try {
op.execute();
}finally {
assertEquals(1L, op.getCounter());
}
}
private static class CountingPageOperation extends AbstractPageOperationTemplate<String>{
private int currentPageFetch;
private int pageSize = 10;
private long counter = 0L;
private long startTime;
private long lastFetchTime;
private CountingPageOperation(int maxPages, long maxTime) {
super(maxPages, maxTime);
startTime = System.currentTimeMillis();
}
@Override
public Collection<String> fetchPage() {
lastFetchTime = System.currentTimeMillis();
List<String> page = new ArrayList<>(pageSize);
for(int i = 0; i < pageSize; i++ ) {
page.add("item " + currentPageFetch * pageSize + i);
}
currentPageFetch++;
return page;
}
@Override
protected void doOperation(String item) {
counter++;
}
public long getCounter() {
return counter;
}
public long getLastFetchTime() {
return lastFetchTime;
}
public long getStartTime(){
return startTime;
}
}
private static class NullPageCountingPageOperation extends CountingPageOperation {
private NullPageCountingPageOperation(int maxPages, long maxTime) {
super(maxPages, maxTime);
}
@Override
public Collection<String> fetchPage() {
return null;
}
}
private static class EmptyPageCountingPageOperation extends CountingPageOperation {
private EmptyPageCountingPageOperation(int maxPages, long maxTime) {
super(maxPages, maxTime);
}
@Override
public Collection<String> fetchPage() {
return new ArrayList<>(0);
}
}
private static class EvenExceptionCountingPageOperation extends CountingPageOperation {
private int callCounter;
private EvenExceptionCountingPageOperation(int maxPages, long maxTime) {
super(maxPages, maxTime);
}
@Override
protected void doOperation(String item) {
callCounter++;
if(callCounter%2 == 0){
throw new IllegalStateException("even number items cannot be processed");
}
super.doOperation(item);
}
}
}

View File

@ -22,6 +22,8 @@ import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.TypedQuery;
import org.mitre.data.DefaultPageCriteria;
import org.mitre.data.PageCriteria;
import org.mitre.oauth2.model.AuthenticationHolderEntity;
import org.mitre.oauth2.repository.AuthenticationHolderRepository;
import org.mitre.util.jpa.JpaUtil;
@ -68,10 +70,15 @@ public class JpaAuthenticationHolderRepository implements AuthenticationHolderRe
@Override
@Transactional(value="defaultTransactionManager")
public List<AuthenticationHolderEntity> getOrphanedAuthenticationHolders() {
TypedQuery<AuthenticationHolderEntity> query = manager.createNamedQuery(AuthenticationHolderEntity.QUERY_GET_UNUSED, AuthenticationHolderEntity.class);
query.setMaxResults(MAXEXPIREDRESULTS);
List<AuthenticationHolderEntity> unusedAuthenticationHolders = query.getResultList();
return unusedAuthenticationHolders;
DefaultPageCriteria pageCriteria = new DefaultPageCriteria(0,MAXEXPIREDRESULTS);
return getOrphanedAuthenticationHolders(pageCriteria);
}
@Override
@Transactional(value="defaultTransactionManager")
public List<AuthenticationHolderEntity> getOrphanedAuthenticationHolders(PageCriteria pageCriteria) {
TypedQuery<AuthenticationHolderEntity> query = manager.createNamedQuery(AuthenticationHolderEntity.QUERY_GET_UNUSED, AuthenticationHolderEntity.class);
return JpaUtil.getResultPage(query, pageCriteria);
}
}

View File

@ -26,6 +26,7 @@ import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.TypedQuery;
import org.mitre.data.PageCriteria;
import org.mitre.oauth2.model.AuthorizationCodeEntity;
import org.mitre.oauth2.repository.AuthorizationCodeRepository;
import org.mitre.util.jpa.JpaUtil;
@ -91,5 +92,13 @@ public class JpaAuthorizationCodeRepository implements AuthorizationCodeReposito
}
@Override
public Collection<AuthorizationCodeEntity> getExpiredCodes(PageCriteria pageCriteria) {
TypedQuery<AuthorizationCodeEntity> query = manager.createNamedQuery(AuthorizationCodeEntity.QUERY_EXPIRATION_BY_DATE, AuthorizationCodeEntity.class);
query.setParameter(AuthorizationCodeEntity.PARAM_DATE, new Date()); // this gets anything that's already expired
return JpaUtil.getResultPage(query, pageCriteria);
}
}

View File

@ -31,6 +31,8 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaDelete;
import javax.persistence.criteria.Root;
import org.mitre.data.DefaultPageCriteria;
import org.mitre.data.PageCriteria;
import org.mitre.oauth2.model.ClientDetailsEntity;
import org.mitre.oauth2.model.OAuth2AccessTokenEntity;
import org.mitre.oauth2.model.OAuth2RefreshTokenEntity;
@ -189,20 +191,32 @@ public class JpaOAuth2TokenRepository implements OAuth2TokenRepository {
@Override
public Set<OAuth2AccessTokenEntity> getAllExpiredAccessTokens() {
TypedQuery<OAuth2AccessTokenEntity> query = manager.createNamedQuery(OAuth2AccessTokenEntity.QUERY_EXPIRED_BY_DATE, OAuth2AccessTokenEntity.class);
query.setParameter(OAuth2AccessTokenEntity.PARAM_DATE, new Date());
query.setMaxResults(MAXEXPIREDRESULTS);
return new LinkedHashSet<>(query.getResultList());
DefaultPageCriteria pageCriteria = new DefaultPageCriteria(0, MAXEXPIREDRESULTS);
return getAllExpiredAccessTokens(pageCriteria);
}
@Override
public Set<OAuth2AccessTokenEntity> getAllExpiredAccessTokens(PageCriteria pageCriteria) {
TypedQuery<OAuth2AccessTokenEntity> query = manager.createNamedQuery(OAuth2AccessTokenEntity.QUERY_EXPIRED_BY_DATE, OAuth2AccessTokenEntity.class);
query.setParameter(OAuth2AccessTokenEntity.PARAM_DATE, new Date());
return new LinkedHashSet<>(JpaUtil.getResultPage(query, pageCriteria));
}
@Override
public Set<OAuth2RefreshTokenEntity> getAllExpiredRefreshTokens() {
TypedQuery<OAuth2RefreshTokenEntity> query = manager.createNamedQuery(OAuth2RefreshTokenEntity.QUERY_EXPIRED_BY_DATE, OAuth2RefreshTokenEntity.class);
query.setParameter(OAuth2RefreshTokenEntity.PARAM_DATE, new Date());
query.setMaxResults(MAXEXPIREDRESULTS);
return new LinkedHashSet<>(query.getResultList());
DefaultPageCriteria pageCriteria = new DefaultPageCriteria(0, MAXEXPIREDRESULTS);
return getAllExpiredRefreshTokens(pageCriteria);
}
@Override
public Set<OAuth2RefreshTokenEntity> getAllExpiredRefreshTokens(PageCriteria pageCriteria) {
TypedQuery<OAuth2RefreshTokenEntity> query = manager.createNamedQuery(OAuth2RefreshTokenEntity.QUERY_EXPIRED_BY_DATE, OAuth2RefreshTokenEntity.class);
query.setParameter(OAuth2AccessTokenEntity.PARAM_DATE, new Date());
return new LinkedHashSet<>(JpaUtil.getResultPage(query,pageCriteria));
}
/* (non-Javadoc)
* @see org.mitre.oauth2.repository.OAuth2TokenRepository#getAccessTokensForResourceSet(org.mitre.uma.model.ResourceSet)
*/

View File

@ -22,6 +22,7 @@ package org.mitre.oauth2.service.impl;
import java.util.Collection;
import java.util.Date;
import org.mitre.data.AbstractPageOperationTemplate;
import org.mitre.oauth2.model.AuthenticationHolderEntity;
import org.mitre.oauth2.model.AuthorizationCodeEntity;
import org.mitre.oauth2.repository.AuthenticationHolderRepository;
@ -116,15 +117,17 @@ public class DefaultOAuth2AuthorizationCodeService implements AuthorizationCodeS
@Transactional(value="defaultTransactionManager")
public void clearExpiredAuthorizationCodes() {
Collection<AuthorizationCodeEntity> codes = repository.getExpiredCodes();
new AbstractPageOperationTemplate<AuthorizationCodeEntity>(){
@Override
public Collection<AuthorizationCodeEntity> fetchPage() {
return repository.getExpiredCodes();
}
for (AuthorizationCodeEntity code : codes) {
repository.remove(code);
}
if (codes.size() > 0) {
logger.info("Removed " + codes.size() + " expired authorization codes.");
}
@Override
protected void doOperation(AuthorizationCodeEntity item) {
repository.remove(item);
}
}.execute();
}
/**

View File

@ -33,6 +33,8 @@ import java.util.List;
import java.util.Set;
import java.util.UUID;
import org.mitre.data.AbstractPageOperationTemplate;
import org.mitre.data.DefaultPageCriteria;
import org.mitre.oauth2.model.AuthenticationHolderEntity;
import org.mitre.oauth2.model.ClientDetailsEntity;
import org.mitre.oauth2.model.OAuth2AccessTokenEntity;
@ -490,47 +492,41 @@ public class DefaultOAuth2ProviderTokenService implements OAuth2TokenEntityServi
public void clearExpiredTokens() {
logger.debug("Cleaning out all expired tokens");
Collection<OAuth2AccessTokenEntity> accessTokens = getExpiredAccessTokens();
if (accessTokens.size() > 0) {
logger.info("Found " + accessTokens.size() + " expired access tokens");
}
for (OAuth2AccessTokenEntity oAuth2AccessTokenEntity : accessTokens) {
try {
revokeAccessToken(oAuth2AccessTokenEntity);
} catch (IllegalArgumentException e) {
//An ID token is deleted with its corresponding access token, but then the ID token is on the list of expired tokens as well and there is
//nothing in place to distinguish it from any other.
//An attempt to delete an already deleted token returns an error, stopping the cleanup dead. We need it to keep going.
}
}
new AbstractPageOperationTemplate<OAuth2AccessTokenEntity>() {
@Override
public Collection<OAuth2AccessTokenEntity> fetchPage() {
return tokenRepository.getAllExpiredAccessTokens(new DefaultPageCriteria());
}
Collection<OAuth2RefreshTokenEntity> refreshTokens = getExpiredRefreshTokens();
if (refreshTokens.size() > 0) {
logger.info("Found " + refreshTokens.size() + " expired refresh tokens");
}
for (OAuth2RefreshTokenEntity oAuth2RefreshTokenEntity : refreshTokens) {
revokeRefreshToken(oAuth2RefreshTokenEntity);
}
@Override
public void doOperation(OAuth2AccessTokenEntity item) {
revokeAccessToken(item);
}
}.execute();
Collection<AuthenticationHolderEntity> authHolders = getOrphanedAuthenticationHolders();
if (authHolders.size() > 0) {
logger.info("Found " + authHolders.size() + " orphaned authentication holders");
}
for(AuthenticationHolderEntity authHolder : authHolders) {
authenticationHolderRepository.remove(authHolder);
}
}
new AbstractPageOperationTemplate<OAuth2RefreshTokenEntity>() {
@Override
public Collection<OAuth2RefreshTokenEntity> fetchPage() {
return tokenRepository.getAllExpiredRefreshTokens(new DefaultPageCriteria());
}
private Collection<OAuth2AccessTokenEntity> getExpiredAccessTokens() {
return Sets.newHashSet(tokenRepository.getAllExpiredAccessTokens());
}
@Override
public void doOperation(OAuth2RefreshTokenEntity item) {
revokeRefreshToken(item);
}
}.execute();
private Collection<OAuth2RefreshTokenEntity> getExpiredRefreshTokens() {
return Sets.newHashSet(tokenRepository.getAllExpiredRefreshTokens());
}
new AbstractPageOperationTemplate<AuthenticationHolderEntity>() {
@Override
public Collection<AuthenticationHolderEntity> fetchPage() {
return authenticationHolderRepository.getOrphanedAuthenticationHolders(new DefaultPageCriteria());
}
private Collection<AuthenticationHolderEntity> getOrphanedAuthenticationHolders() {
return Sets.newHashSet(authenticationHolderRepository.getOrphanedAuthenticationHolders());
@Override
public void doOperation(AuthenticationHolderEntity item) {
authenticationHolderRepository.remove(item);
}
}.execute();
}
/* (non-Javadoc)