From 78532106a30e92eb90ce4e5f16ab5bf1dad6f6f0 Mon Sep 17 00:00:00 2001 From: Qiuyi LI Date: Fri, 28 Jul 2023 15:32:17 +0200 Subject: [PATCH] Create HLJob for parcel traces --- .../modules/business/domain/job/HLJob.java | 124 ++++++++++++++++++ .../modules/business/mapper/ParcelMapper.java | 2 + .../business/mapper/ParcelTraceMapper.java | 3 + .../business/mapper/xml/ParcelMapper.xml | 21 +++ .../business/mapper/xml/ParcelTraceMapper.xml | 22 ++++ .../business/service/IParcelService.java | 3 + .../service/impl/ParcelServiceImpl.java | 42 ++++++ 7 files changed, 217 insertions(+) create mode 100644 jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/HLJob.java diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/HLJob.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/HLJob.java new file mode 100644 index 000000000..27e776f7d --- /dev/null +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/HLJob.java @@ -0,0 +1,124 @@ +package org.jeecg.modules.business.domain.job; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpEntity; +import org.apache.http.util.EntityUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.jeecg.modules.business.domain.api.hualei.HLRequest; +import org.jeecg.modules.business.domain.api.hualei.HLResponse; +import org.jeecg.modules.business.domain.api.hualei.HLResponseItem; +import org.jeecg.modules.business.service.IParcelService; +import org.jeecg.modules.business.service.IPlatformOrderService; +import org.quartz.Job; +import org.quartz.JobDataMap; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.sql.Date; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Slf4j +public class HLJob implements Job { + + @Autowired + private IParcelService parcelService; + @Autowired + private IPlatformOrderService platformOrderService; + + private static final Integer DEFAULT_NUMBER_OF_DAYS = 15; + private static final Integer DEFAULT_MAXIMUM_NUMBER_OF_PARCELS_PER_TRANSACTION = 800; + private static final List DEFAULT_TRANSPORTERS = Arrays.asList("法国专线普货"); + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + LocalDate endDate = LocalDate.now(); + LocalDate startDate = endDate.minusDays(DEFAULT_NUMBER_OF_DAYS); + List transporters = DEFAULT_TRANSPORTERS; + boolean overrideRestriction = false; + ObjectMapper mapper = new ObjectMapper(); + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + JobDataMap jobDataMap = context.getMergedJobDataMap(); + String parameter = ((String) jobDataMap.get("parameter")); + if (parameter != null) { + try { + JSONObject jsonObject = new JSONObject(parameter); + if (!jsonObject.isNull("startDate")) { + String startDateStr = jsonObject.getString("startDate"); + startDate = LocalDate.parse(startDateStr); + } + if (!jsonObject.isNull("endDate")) { + String endDateStr = jsonObject.getString("endDate"); + endDate = LocalDate.parse(endDateStr); + } + if (!jsonObject.isNull("transporters")) { + JSONArray transporterArray = jsonObject.getJSONArray("transporters"); + List transporterList = new ArrayList<>(); + for (int i = 0; i < transporterArray.length(); i++) { + transporterList.add(transporterArray.getString(i)); + } + transporters = transporterList; + } + if (!jsonObject.isNull("override")) { + overrideRestriction = jsonObject.getBoolean("override"); + } + } catch (JSONException e) { + log.error("Error while parsing parameter as JSON, falling back to default parameters."); + } + } + + if (!endDate.isAfter(startDate)) { + throw new RuntimeException("EndDate must be strictly greater than StartDate !"); + } else if (endDate.minusDays(30).isAfter(startDate) && !overrideRestriction) { + throw new RuntimeException("No more than 30 days can separate startDate and endDate !"); + } + + log.info("Starting to retrieve parcel traces of {} from {} to {}", transporters, startDate, endDate); + List billCodes = platformOrderService.fetchBillCodesOfParcelsWithoutTrace( + Date.valueOf(startDate), Date.valueOf(endDate), transporters); + log.info("{} parcels without trace in total", billCodes.size()); + List> billCodeLists = Lists.partition(billCodes, 40); + log.info("Requests will be divided in to {} parts", billCodeLists.size()); + List parcelTraces = new ArrayList<>(); + List hlRequests = new ArrayList<>(); + billCodeLists.forEach(billcodeList -> { + HLRequest hlRequest = new HLRequest(billcodeList); + hlRequests.add(hlRequest); + }); + List results = new ArrayList<>(); + for (HLRequest request : hlRequests) { + boolean success = false; + HttpEntity entity = request.send().getEntity(); + try { + // String of the response + String responseString = EntityUtils.toString(entity, "UTF-8"); + // Geniuses at HL decided to put the sole response object in an array.... + HLResponse[] hlResponseArray = mapper.readValue(responseString, HLResponse[].class); + parcelTraces.addAll(hlResponseArray[0].getResponseItems()); + success = true; + } catch (IOException e) { + log.error("Error while parsing response into String", e); + } finally { + results.add(success); + } + log.info("{} parcel added to the queue to be inserted into DB.", parcelTraces.size()); + } + long nbSuccesses = results.stream().filter(b -> b).count(); + log.info("{}/{} lots of 40 parcel traces have been retrieved.", nbSuccesses, hlRequests.size()); + + List> parcelTraceList = Lists.partition(parcelTraces, DEFAULT_MAXIMUM_NUMBER_OF_PARCELS_PER_TRANSACTION); + for (List parcelTracesPerTransaction : parcelTraceList) { + parcelService.saveHLParcelAndTraces(parcelTracesPerTransaction); + } + } + +} diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelMapper.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelMapper.java index ae6c8b825..4b2d3081b 100644 --- a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelMapper.java +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelMapper.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.Param; import org.jeecg.modules.business.domain.api.cmk.CMKParcelTraceData; import org.jeecg.modules.business.domain.api.equick.EQuickResponse; +import org.jeecg.modules.business.domain.api.hualei.HLResponseItem; import org.jeecg.modules.business.domain.api.jt.JTParcelTrace; import org.jeecg.modules.business.domain.api.yd.YDTraceData; import org.jeecg.modules.business.entity.Parcel; @@ -28,6 +29,7 @@ public interface ParcelMapper extends BaseMapper { void insertOrIgnoreYDParcels(List parcels); void insertOrIgnoreCMKParcels(List parcels); + void insertOrIgnoreHLParcels(List parcels); /** * fetch all parcels from platform order's tracking number, to archive diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelTraceMapper.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelTraceMapper.java index d7f0f797e..b507694e1 100644 --- a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelTraceMapper.java +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/ParcelTraceMapper.java @@ -5,6 +5,7 @@ import org.apache.ibatis.annotations.Param; import org.jeecg.modules.business.domain.api.cmk.CMKParcelTrace; import org.jeecg.modules.business.domain.api.cmk.CMKParcelTraceData; import org.jeecg.modules.business.domain.api.equick.EQuickTraceData; +import org.jeecg.modules.business.domain.api.hualei.HLParcelTraceDetail; import org.jeecg.modules.business.domain.api.jt.JTParcelTraceDetail; import org.jeecg.modules.business.domain.api.yd.YDTraceDetail; import org.jeecg.modules.business.entity.ParcelTrace; @@ -33,6 +34,8 @@ public interface ParcelTraceMapper extends BaseMapper { void insertOrIgnoreCMKTraces(@Param("traces") List traceDetails); + void insertOrIgnoreHLTraces(@Param("traces") List tracesToInsert); + /** * Fetches parcel traces to archive * @param parcelIDs diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/xml/ParcelMapper.xml b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/xml/ParcelMapper.xml index c745bb1bd..35471682c 100644 --- a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/xml/ParcelMapper.xml +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/xml/ParcelMapper.xml @@ -86,6 +86,27 @@ ) + + + INSERT IGNORE INTO parcel(id, create_by, create_time, update_by, update_time, bill_code, country, + third_bill_code, order_no, product_code) + VALUES + + ( + #{parcel.id}, + 'hl api', + NOW(), + 'hl api', + NOW(), + #{parcel.billCode}, + #{parcel.country}, + #{parcel.trackingNumber}, + #{parcel.orderNumber}, + #{parcel.productCode} + ) + + + SELECT * FROM parcel_trace diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/IParcelService.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/IParcelService.java index 8eee3e9de..2c06d3270 100644 --- a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/IParcelService.java +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/IParcelService.java @@ -3,6 +3,7 @@ package org.jeecg.modules.business.service; import com.baomidou.mybatisplus.extension.service.IService; import org.jeecg.modules.business.domain.api.cmk.CMKParcelTraceData; import org.jeecg.modules.business.domain.api.equick.EQuickResponse; +import org.jeecg.modules.business.domain.api.hualei.HLResponseItem; import org.jeecg.modules.business.domain.api.jt.JTParcelTrace; import org.jeecg.modules.business.domain.api.yd.YDTraceData; import org.jeecg.modules.business.entity.Parcel; @@ -48,6 +49,8 @@ public interface IParcelService extends IService { void saveYDParcelAndTraces(List traceData); void saveCMKParcelAndTraces(List traceData); + + void saveHLParcelAndTraces(List parcelTraces); List fetchParcelsToArchive(List trackingNumbers); void saveParcelArchive(List parcels); } diff --git a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/impl/ParcelServiceImpl.java b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/impl/ParcelServiceImpl.java index e65b25397..c2a73badd 100644 --- a/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/impl/ParcelServiceImpl.java +++ b/jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/impl/ParcelServiceImpl.java @@ -7,6 +7,8 @@ import org.jeecg.modules.business.domain.api.cmk.CMKParcelTrace; import org.jeecg.modules.business.domain.api.cmk.CMKParcelTraceData; import org.jeecg.modules.business.domain.api.equick.EQuickResponse; import org.jeecg.modules.business.domain.api.equick.EQuickTraceData; +import org.jeecg.modules.business.domain.api.hualei.HLParcelTraceDetail; +import org.jeecg.modules.business.domain.api.hualei.HLResponseItem; import org.jeecg.modules.business.domain.api.jt.JTParcelTrace; import org.jeecg.modules.business.domain.api.jt.JTParcelTraceDetail; import org.jeecg.modules.business.domain.api.yd.YDTraceData; @@ -272,6 +274,46 @@ public class ParcelServiceImpl extends ServiceImpl impleme } log.info("Finished inserting {} parcels and their traces into DB.", parcelTraces.size()); } + + @Override + @Transactional + public void saveHLParcelAndTraces(List parcelTraces) { + if (parcelTraces.isEmpty()) { + return; + } + log.info("Started inserting {} HL parcels and their traces into DB.", parcelTraces.size() ); + List parcelBillCodes = parcelTraces.stream() + .map(HLResponseItem::getBillCode) + .collect(Collectors.toList()); + List existingParcels = parcelMapper.searchByBillCode(parcelBillCodes); + Map billCodeToExistingParcels = existingParcels.stream().collect( + Collectors.toMap(Parcel::getBillCode, Function.identity()) + ); + List parcelToInsert = new ArrayList<>(); + List tracesToInsert = new ArrayList<>(); + for (HLResponseItem parcelAndTrace : parcelTraces) { + List traceDetails = parcelAndTrace.getTracesList(); + if (traceDetails.isEmpty()) { + break; + } + Parcel existingParcel = billCodeToExistingParcels.get(parcelAndTrace.getBillCode()); + if (existingParcel == null) { + parcelToInsert.add(parcelAndTrace); + traceDetails.forEach(trace -> trace.parcelTraceProcess(parcelAndTrace.getId())); + } else { + traceDetails.forEach(trace -> trace.parcelTraceProcess(existingParcel.getId())); + } + tracesToInsert.addAll(traceDetails); + } + log.info("After filtering, {} parcels will be inserted into the DB.", parcelToInsert.size()); + if (!parcelToInsert.isEmpty()) { + parcelMapper.insertOrIgnoreHLParcels(parcelToInsert); + } + if (!tracesToInsert.isEmpty()) { + parcelTraceMapper.insertOrIgnoreHLTraces(tracesToInsert); + } + log.info("Finished inserting {} HL parcels and their traces into DB.", parcelTraces.size()); + } @Override @Transactional public List fetchParcelsToArchive(List trackingNumbers) {