From bbd148cb5182c8c8cb153cbe921232c84ab72da4 Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Tue, 5 Mar 2024 15:53:11 +0100 Subject: [PATCH 1/7] Apply a throttler of 40 requests per minute to ShopifySyncJob (cherry picked from commit 188c5601bd151381afbfdc9fb65370c082485949) --- .../modules/business/domain/job/ShopifySyncJob.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/ShopifySyncJob.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/ShopifySyncJob.java index 1c0b7b3fc..e370c41c1 100644 --- a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/ShopifySyncJob.java +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/ShopifySyncJob.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -33,6 +34,7 @@ public class ShopifySyncJob implements Job { private static final List DEFAULT_INCLUDED_SHOPS = Arrays.asList("JCH3", "JCH4", "JCH5"); private static final Integer DEFAULT_NUMBER_OF_THREADS = 10; + private static final Integer SHOPIFY_API_RATE_LIMIT_PER_SHOP_PER_MINUTE = 40; @Autowired private IPlatformOrderService platformOrderService; @@ -66,7 +68,8 @@ public class ShopifySyncJob implements Job { List fulfillmentOrders = new ArrayList<>(); - ExecutorService executor = Executors.newFixedThreadPool(DEFAULT_NUMBER_OF_THREADS); + ExecutorService throttlingExecutorService = ThrottlingExecutorService.createExecutorService(DEFAULT_NUMBER_OF_THREADS, + SHOPIFY_API_RATE_LIMIT_PER_SHOP_PER_MINUTE, TimeUnit.MINUTES); log.info("Constructing fulfillment retrieval requests"); List getFulfillmentRequestBodyList = new ArrayList<>(); ordersReadyForShopifySync.forEach(o -> getFulfillmentRequestBodyList.add(new GetFulfillmentRequestBody(o))); @@ -88,7 +91,7 @@ public class ShopifySyncJob implements Job { log.error("Error processing json", e); } return success; - }, executor)) + }, throttlingExecutorService)) .collect(Collectors.toList()); List results = futures.stream().map(CompletableFuture::join).collect(Collectors.toList()); long nbSuccesses = results.stream().filter(b -> b).count(); @@ -120,7 +123,7 @@ public class ShopifySyncJob implements Job { log.error("Error processing json", e); } return success; - }, executor)) + }, throttlingExecutorService)) .collect(Collectors.toList()); results = fulfillmentCreationFutures.stream().map(CompletableFuture::join).collect(Collectors.toList()); nbSuccesses = results.stream().filter(b -> b).count(); From c52c1159429b7a0d5866e29bf5a593264041a593 Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Tue, 5 Mar 2024 16:32:04 +0100 Subject: [PATCH 2/7] Update project version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d8a7c2a40..6b6b2aaf9 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.jeecgframework.boot jeecg-boot-parent - 1.10.0 + 1.10.1 pom WIA APP ${project.version} From f0bbb1c02c389bf0745aeebc055da19344ba0ba5 Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Tue, 12 Mar 2024 12:14:35 +0100 Subject: [PATCH 3/7] Update CJ_LOGISTICS tracking number pattern --- .../business/domain/api/shopify/CreateFulfillmentRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/shopify/CreateFulfillmentRequest.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/shopify/CreateFulfillmentRequest.java index 23c8808a3..e5e24a19d 100644 --- a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/shopify/CreateFulfillmentRequest.java +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/shopify/CreateFulfillmentRequest.java @@ -43,7 +43,7 @@ public class CreateFulfillmentRequest extends ShopifyRequest { POST_NL("https://postnl.post/", "PostNL International Mail", "LS[0-9]{9}NL"), COLI_COLI("https://www.colicoli.fr/trackings?id=%s", "Coli Coli", "CC[0-9]{14}[A-Z]*"), LUXEMBOURG_POST("https://www.post.lu/particuliers/colis-courrier/track-and-trace#/search", "Luxembourg Post", "LL[0-9]{9}LU"), - CJ_LOGISTICS("https://www.cjlogistics.com/ko/tool/parcel/tracking", "CJ대한통운", "57575[0-9]{7}"), + CJ_LOGISTICS("https://www.cjlogistics.com/ko/tool/parcel/tracking", "CJ대한통운", "57575[0-9]{7}|58476[0-9]{7}"), ; private final String trackingUrl; From 06228752d8d25567248753d265403eeb380ac928 Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Tue, 12 Mar 2024 12:15:05 +0100 Subject: [PATCH 4/7] Update project version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6b6b2aaf9..9ca41c38b 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.jeecgframework.boot jeecg-boot-parent - 1.10.1 + 1.10.2 pom WIA APP ${project.version} From 0a372b09948901f6ce660f6c9c3ca7d5728eaaec Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Thu, 14 Mar 2024 16:06:12 +0100 Subject: [PATCH 5/7] Create a job to automatically remove virtual products from Mabang orders --- .../dochangeorder/RemoveSkuRequest.java | 15 ++ .../dochangeorder/RemoveSkuRequestBody.java | 53 ++++++ .../domain/job/RemoveVirtualProductJob.java | 154 ++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequest.java create mode 100644 jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequestBody.java create mode 100644 jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/RemoveVirtualProductJob.java diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequest.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequest.java new file mode 100644 index 000000000..65075bc76 --- /dev/null +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequest.java @@ -0,0 +1,15 @@ +package org.jeecg.modules.business.domain.api.mabang.dochangeorder; + +import org.jeecg.modules.business.domain.api.mabang.Request; + +public class RemoveSkuRequest extends Request { + public RemoveSkuRequest(RemoveSkuRequestBody body) { + super(body); + } + + @Override + public ChangeOrderResponse send() { + String jsonString = rawSend().getBody(); + return ChangeOrderResponse.parse(jsonString); + } +} diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequestBody.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequestBody.java new file mode 100644 index 000000000..5e53f41d9 --- /dev/null +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/dochangeorder/RemoveSkuRequestBody.java @@ -0,0 +1,53 @@ +package org.jeecg.modules.business.domain.api.mabang.dochangeorder; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import lombok.Data; +import org.apache.commons.lang3.tuple.Pair; +import org.jeecg.modules.business.domain.api.mabang.RequestBody; + +import java.util.HashSet; + +@Data +public class RemoveSkuRequestBody implements RequestBody { + + + private String platformOrderId; + private final HashSet> virtualSkus; + private final static String DEFAULT_WAREHOUSE_NAME = "SZBA宝安仓"; + + public RemoveSkuRequestBody(String platformOrderId, HashSet> virtualSkus) { + this.platformOrderId = platformOrderId; + this.virtualSkus = virtualSkus; + } + + @Override + public String api() { + return "order-do-change-order"; + } + + @Override + public JSONObject parameters() { + JSONObject json = new JSONObject(); + putNonNull(json, "platformOrderId", platformOrderId); + JSONArray stockDataArray = new JSONArray(); + if (!virtualSkus.isEmpty()) { + for (Pair virtualSku : virtualSkus) { + JSONObject stockData = new JSONObject(); + stockData.put("warehouseName", DEFAULT_WAREHOUSE_NAME); + stockData.put("stockSku", virtualSku.getKey()); + stockData.put("quantity", virtualSku.getValue()); + stockData.put("type", 2); + stockDataArray.add(stockData); + } + } + json.put("stockData", stockDataArray.toJSONString()); + return json; + } + + private void putNonNull(JSONObject json, String key, E value) { + if (value != null) { + json.put(key, value); + } + } +} diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/RemoveVirtualProductJob.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/RemoveVirtualProductJob.java new file mode 100644 index 000000000..877ab0680 --- /dev/null +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/RemoveVirtualProductJob.java @@ -0,0 +1,154 @@ +package org.jeecg.modules.business.domain.job; + +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.jeecg.modules.business.domain.api.mabang.dochangeorder.ChangeOrderResponse; +import org.jeecg.modules.business.domain.api.mabang.dochangeorder.RemoveSkuRequest; +import org.jeecg.modules.business.domain.api.mabang.dochangeorder.RemoveSkuRequestBody; +import org.jeecg.modules.business.domain.api.mabang.getorderlist.*; +import org.jeecg.modules.business.service.IPlatformOrderService; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.beans.factory.annotation.Autowired; + +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Slf4j +public class RemoveVirtualProductJob implements Job { + + private static final Integer DEFAULT_NUMBER_OF_DAYS = 5; + + private static final Integer DEFAULT_NUMBER_OF_THREADS = 10; + private static final String REMOVED_SKU_STATUS = "4";; + private Map> virtualSkusByShop = new HashMap<>(); + + @Autowired + private IPlatformOrderService platformOrderService; + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + LocalDateTime endDateTime = LocalDateTime.now(); + LocalDateTime startDateTime = endDateTime.minusDays(DEFAULT_NUMBER_OF_DAYS); + List shops = new ArrayList<>(); + JobDataMap jobDataMap = context.getMergedJobDataMap(); + String parameter = ((String) jobDataMap.get("parameter")); + if (parameter != null) { + try { + JSONObject jsonObject = new JSONObject(parameter); + if (!jsonObject.isNull("startDateTime")) { + String startDateStr = jsonObject.getString("startDateTime"); + startDateTime = LocalDateTime.parse(startDateStr); + } + if (!jsonObject.isNull("endDateTime")) { + String endDateStr = jsonObject.getString("endDateTime"); + endDateTime = LocalDateTime.parse(endDateStr); + } + if (!jsonObject.isNull("virtualSkusByShop")) { + JSONArray virtualSkusByShopArray = jsonObject.getJSONArray("virtualSkusByShop"); + for (int i = 0; i < virtualSkusByShopArray.length(); i++) { + JSONObject object = virtualSkusByShopArray.getJSONObject((i)); + if (!object.isNull("shop")) { + String shopCode = object.getString("shop"); + shops.add(shopCode); + if (!object.isNull("virtualSkus")) { + JSONArray virtualSkusArray = object.getJSONArray("virtualSkus"); + List virtualSkus = new ArrayList<>(); + for (int j = 0; j < virtualSkusArray.length(); j++) { + virtualSkus.add(virtualSkusArray.getString(j)); + } + virtualSkusByShop.put(shopCode, virtualSkus); + } + } + } + } + } catch (JSONException e) { + log.error("Error while parsing parameter as JSON, falling back to default parameters."); + } + } + + if (!endDateTime.isAfter(startDateTime)) { + throw new RuntimeException("EndDateTime must be strictly greater than StartDateTime !"); + } + + List platformOrderIds = platformOrderService.fetchUninvoicedOrdersForShops(startDateTime, endDateTime, shops); + List> platformOrderIdLists = Lists.partition(platformOrderIds, 10); + + List requests = new ArrayList<>(); + for (List platformOrderIdList : platformOrderIdLists) { + requests.add(new OrderListRequestBody().setPlatformOrderIds(platformOrderIdList)); + } + List mabangOrders = new ArrayList<>(); + + ExecutorService executor = Executors.newFixedThreadPool(DEFAULT_NUMBER_OF_THREADS); + List> futures = requests.stream() + .map(request -> CompletableFuture.supplyAsync(() -> { + boolean success = false; + try { + OrderListRawStream rawStream = new OrderListRawStream(request); + OrderListStream stream = new OrderListStream(rawStream); + List orders = stream.all(); + mabangOrders.addAll(orders); + success = !orders.isEmpty(); + } catch (RuntimeException e) { + log.error("Error communicating with MabangAPI", e); + } + return success; + }, executor)) + .collect(Collectors.toList()); + List results = futures.stream().map(CompletableFuture::join).collect(Collectors.toList()); + long nbSuccesses = results.stream().filter(b -> b).count(); + log.info("{}/{} requests have succeeded.", nbSuccesses, requests.size()); + log.info("{}/{} mabang orders have been retrieved.", mabangOrders.size(), platformOrderIds.size()); + + log.info("Constructing virtual SKU removal requests"); + List removeSkuRequests = new ArrayList<>(); + Set shopErpCodes = virtualSkusByShop.keySet(); + for (Order mabangOrder : mabangOrders) { + String shopErpCode = mabangOrder.getShopErpCode(); + if (shopErpCodes.contains(shopErpCode)) { + List virtualSkus = virtualSkusByShop.get(shopErpCode); + HashSet> virtualSkuToRemove = new HashSet<>(); + for (OrderItem orderItem : mabangOrder.getOrderItems()) { + String skuErpCode = orderItem.getErpCode(); + if (!orderItem.getStatus().equalsIgnoreCase(REMOVED_SKU_STATUS) && virtualSkus.contains(skuErpCode)) { + virtualSkuToRemove.add(Pair.of(skuErpCode, orderItem.getQuantity())); + } + } + if (!virtualSkuToRemove.isEmpty()) { + RemoveSkuRequestBody removeSkuRequestBody = new RemoveSkuRequestBody(mabangOrder.getPlatformOrderId(), + virtualSkuToRemove); + removeSkuRequests.add(removeSkuRequestBody); + } + } + } + log.info("{} virtual SKU removal requests to be sent to MabangAPI", removeSkuRequests.size()); + + List> removeSkuFutures = removeSkuRequests.stream() + .map(removeSkuRequestBody -> CompletableFuture.supplyAsync(() -> { + boolean success = false; + try { + RemoveSkuRequest changeOrderRequest = new RemoveSkuRequest(removeSkuRequestBody); + ChangeOrderResponse response = changeOrderRequest.send(); + success = response.success(); + } catch (RuntimeException e) { + log.error("Error communicating with MabangAPI", e); + } + return success; + }, executor)) + .collect(Collectors.toList()); + results = removeSkuFutures.stream().map(CompletableFuture::join).collect(Collectors.toList()); + nbSuccesses = results.stream().filter(b -> b).count(); + log.info("{}/{} virtual SKU removal requests have succeeded.", nbSuccesses, removeSkuRequests.size()); + } +} From c435377fcacacd514096bde5928889a069cd8ee4 Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Thu, 14 Mar 2024 16:06:58 +0100 Subject: [PATCH 6/7] Update project version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9ca41c38b..89fb9a8d8 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.jeecgframework.boot jeecg-boot-parent - 1.10.2 + 1.10.3 pom WIA APP ${project.version} From 7d5219715b3df595694f49ddf220b07f7436c336 Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Thu, 28 Mar 2024 11:40:49 +0100 Subject: [PATCH 7/7] Release 1.11.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 89fb9a8d8..d2d951b3f 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 org.jeecgframework.boot jeecg-boot-parent - 1.10.3 + 1.11.0 pom WIA APP ${project.version}