Merge pull request #94 from LQYBill/feat/updateSkuJob

feat: sku update job

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/doSearchSkuListNew/SkuData.java

@@ -90,6 +90,9 @@ public class SkuData {
     public SkuStatus getStatus() {
         return SkuStatus.fromCode(this.status);
     }
+    public int getStatusValue() {
+        return this.status;
+    }
     public String toString() {
         return "ID : " + this.id +
                 "\nStockSkuId : " + this.stockSkuId +

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/doSearchSkuListNew/SkuListRequestBody.java

@@ -18,7 +18,7 @@ public class SkuListRequestBody implements RequestBody {
     private String stockSku = null;
     // 50 skus max
     private String stockSkuList = null;
-    private DateType datetimeType;
+    private DateType datetimeType = DateType.CREATE;
     private LocalDateTime startDate;
     private LocalDateTime endDate;
     private Integer page = 1;

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/api/mabang/doSearchSkuListNew/SkuUpdateListStream.java

@@ -0,0 +1,122 @@
+package org.jeecg.modules.business.domain.api.mabang.doSearchSkuListNew;
+
+import lombok.extern.slf4j.Slf4j;
+import org.jeecg.modules.business.domain.api.mabang.getorderlist.NetworkDataStream;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * This class provide stream of order.
+ */
+@Slf4j
+public class SkuUpdateListStream implements NetworkDataStream<SkuData> {
+
+    private final NetworkDataStream<SkuListResponse> rawStream;
+
+    private List<SkuData> skus;
+
+    private int index;
+
+    private boolean began;
+
+    /**
+     * Flag of current data is already empty,
+     * either currentOrders is null or currentIndex arrives at the end.
+     * In both case, we should call next() of the rawStream.
+     */
+    private boolean empty;
+
+
+    public SkuUpdateListStream(NetworkDataStream<SkuListResponse> rawStream) {
+        this.rawStream = rawStream;
+        skus = null;
+        this.index = 0;
+        this.empty = true;
+        this.began = false;
+    }
+    @Override
+    public List<SkuData> all() {
+        SkuData firstElement = attempt();
+        if (firstElement == null) {
+            return Collections.emptyList();
+        }
+
+        ArrayList<SkuData> res = new ArrayList<>();
+        if (firstElement.getStatus().equals(SkuStatus.Normal) || firstElement.getStatus().equals(SkuStatus.StoppedSelling)) {
+            res.add(firstElement);
+        }
+        while (hasNext()) {
+            SkuData nextSku = next();
+            if(nextSku.getStatus().equals(SkuStatus.Normal) || firstElement.getStatus().equals(SkuStatus.StoppedSelling)) {
+                res.add(nextSku);
+            }
+        }
+        return res;
+    }
+    @Override
+    public SkuData attempt() {
+        began = true;
+        log.info("Attempting for the first request");
+        SkuListResponse response = rawStream.attempt();
+        if (response == null) {
+            log.info("No response");
+            return null;
+        }
+        if (response.getData().isEmpty()) {
+            log.info("Response with empty data");
+            return null;
+        }
+        skus = response.getData().toJavaList(SkuData.class);
+        index = 1;
+        log.info("Returned the first element");
+        empty = index >= skus.size();
+        return skus.get(0);
+    }
+
+    @Override
+    public boolean hasNext() {
+        // the first time
+        if (!began) {
+            throw new IllegalStateException("Calling hasNext before begin");
+        }
+
+        // Current data is not yet empty
+        if (index < skus.size()) {
+            log.debug("Current order list is not empty yet");
+            return true;
+        }
+
+        /* Current data is empty */
+        this.empty = true;
+        log.debug("Current order list is already empty,");
+        // and raw stream is empty too.
+        if (!rawStream.hasNext()) {
+            log.debug("and source stream is empty too, hasNext: false");
+            return false;
+        }
+        // but raw stream not empty.
+        else {
+            log.debug("but source stream still has data, hasNext: true");
+            return true;
+        }
+    }
+
+    @Override
+    public SkuData next() {
+        if (!hasNext()) {
+            throw new NoSuchElementException("Stream is empty!");
+        }
+        if (empty) {
+            skus = this.rawStream.next().getData().toJavaList(SkuData.class);
+            empty = false;
+            index = 0;
+        }
+        log.debug("Return data at {}", index);
+        SkuData res = skus.get(index);
+        index++;
+        return res;
+    }
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/domain/job/MabangSkuSyncJob.java

@@ -0,0 +1,113 @@
+package org.jeecg.modules.business.domain.job;
+
+import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.jeecg.modules.business.domain.api.mabang.doSearchSkuListNew.*;
+import org.jeecg.modules.business.service.ISkuListMabangService;
+import org.quartz.Job;
+import org.quartz.JobDataMap;
+import org.quartz.JobExecutionContext;
+import org.quartz.JobExecutionException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A Job that retrieves all Sku from Mabang
+ * if the sku is of status 3 (normal) and not in DB, then we insert it in DB
+ */
+@Slf4j
+@Component
+public class MabangSkuSyncJob implements Job {
+
+    @Autowired
+    private ISkuListMabangService skuListMabangService;
+    private static final Integer DEFAULT_NUMBER_OF_DAYS = 5;
+    private static final DateType DEFAULT_DATE_TYPE = DateType.UPDATE;
+
+    @Override
+    public void execute(JobExecutionContext context) throws JobExecutionException {
+        LocalDateTime endDateTime = LocalDateTime.now(ZoneId.of(ZoneId.SHORT_IDS.get("CTT")));
+        LocalDateTime startDateTime = endDateTime.minusDays(DEFAULT_NUMBER_OF_DAYS);
+        List<String> skus = new ArrayList<>();
+        DateType dateType = DEFAULT_DATE_TYPE;
+        JobDataMap jobDataMap = context.getMergedJobDataMap();
+        String parameter = ((String) jobDataMap.get("parameter"));
+        if (parameter != null) {
+            try {
+                JSONObject jsonObject = new JSONObject(parameter);
+                if (!jsonObject.isNull("startDateTime")) {
+                    String startDateStr = jsonObject.getString("startDateTime");
+                    startDateTime = LocalDateTime.parse(startDateStr);
+                }
+                if (!jsonObject.isNull("endDateTime")) {
+                    String endDateStr = jsonObject.getString("endDateTime");
+                    endDateTime = LocalDateTime.parse(endDateStr);
+                }
+                if (!jsonObject.isNull("dateType")) {
+                    dateType = DateType.fromCode(jsonObject.getInt("dateType"));
+                }
+                if (!jsonObject.isNull("skus")) {
+                    JSONArray array = jsonObject.getJSONArray("skus");
+                    for(int i = 0; i < array.length(); i++) {
+                        skus.add(array.getString(i));
+                    }
+                }
+            } catch (JSONException e) {
+                log.error("Error while parsing parameter as JSON, falling back to default parameters.");
+            }
+        }
+
+        if (!endDateTime.isAfter(startDateTime)) {
+            throw new RuntimeException("EndDateTime must be strictly greater than StartDateTime !");
+        }
+
+        try {
+            if(skus.isEmpty()) {
+                log.info("Updating skus by date");
+                while (startDateTime.until(endDateTime, ChronoUnit.HOURS) > 0) {
+                    LocalDateTime dayBeforeEndDateTime = endDateTime.minusDays(1);
+                    SkuListRequestBody body = SkuListRequestBodys.allSkuOfDateType(dayBeforeEndDateTime, endDateTime, dateType);
+                    SkuListRawStream rawStream = new SkuListRawStream(body);
+                    SkuUpdateListStream stream = new SkuUpdateListStream(rawStream);
+                    // the status is directly filtered in all() method
+                    List<SkuData> skusFromMabang = stream.all();
+                    log.info("{} skus from {} to {} ({})to be updated.", skusFromMabang.size(),
+                            dayBeforeEndDateTime, endDateTime, dateType);
+
+                    if (!skusFromMabang.isEmpty()) {
+                        // we save the skuDatas in DB
+                        skuListMabangService.updateSkusFromMabang(skusFromMabang);
+                    }
+                    endDateTime = dayBeforeEndDateTime;
+                }
+            }
+            else {
+                log.info("Updating skus by erpCode : {}", skus);
+                List<List<String>> skusPartition = Lists.partition(skus, 50);
+                for(List<String> skuPartition : skusPartition) {
+                    SkuListRequestBody body = new SkuListRequestBody();
+                    body.setStockSkuList(String.join(",", skuPartition));
+                    SkuListRawStream rawStream = new SkuListRawStream(body);
+                    SkuUpdateListStream stream = new SkuUpdateListStream(rawStream);
+                    List<SkuData> skusFromMabang = stream.all();
+                    log.info("{} skus to be updated.", skusFromMabang.size());
+                    if (!skusFromMabang.isEmpty()) {
+                        // we save the skuDatas in DB
+                        skuListMabangService.updateSkusFromMabang(skusFromMabang);
+                    }
+                }
+            }
+        } catch (SkuListRequestErrorException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/entity/Sku.java

@@ -98,4 +98,12 @@ public class Sku implements Serializable {
     @Excel(name = "服务费", width = 15)
     @ApiModelProperty(value = "服务费")
     private java.math.BigDecimal serviceFee;
+    /**
+     * Status
+     * 1:自动创建;2:待开发;3:正常;4:清仓;5:停止销售"
+     * default : 3
+     */
+    @Excel(name = "Status", width = 15)
+    @ApiModelProperty(value = "Status")
+    private java.lang.Integer status;
 }

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/mapper/xml/SkuMapper.xml

@@ -209,6 +209,7 @@
                  LEFT JOIN sales_7 s7 ON s.id = s7.sku_id
                  LEFT JOIN qtyInOrdersNotShipped ON s.id = qtyInOrdersNotShipped.ID
         WHERE client_sku.client_id = #{clientId}
+            AND s.status = 3
         ORDER BY ${column} ${order}
         <if test="size != -1">
             LIMIT #{offset}, #{size}
@@ -272,6 +273,7 @@
                  LEFT JOIN sales_7 s7 ON s.id = s7.sku_id
                  LEFT JOIN qtyInOrdersNotShippedCTE ON s.id = qtyInOrdersNotShippedCTE.ID
         WHERE client_sku.client_id = #{clientId}
+            AND s.status = 3
             AND (
                 <if test="erpCodes != ''">
                     s.erp_code REGEXP #{erpCodes}

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/ISkuListMabangService.java

@@ -15,6 +15,7 @@ public interface ISkuListMabangService extends IService<SkuData> {
      * @param skuDataList skus to save.
      */
     Map<Sku, String> saveSkuFromMabang(List<SkuData> skuDataList);
+    void updateSkusFromMabang(List<SkuData> skuDataList);
 
     /**
      * Save products to DB from mabang api.

jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/business/service/impl/SkuListMabangServiceImpl.java

@@ -54,7 +54,10 @@ public class SkuListMabangServiceImpl extends ServiceImpl<SkuListMabangMapper, S
     @Autowired
     Environment env;
 
-
+    // In NameCN field on top of the product name we also get the customer code in the beginning of the string : "XX Description of the product"
+    final Pattern cnNamePattern = Pattern.compile("^([a-zA-Z]{2,5})\\s(.*)$");
+    // In NameEN field on top of the product name we also get the customer code in the beginning of the string : "XX-En name of product"
+    final Pattern enNamePattern = Pattern.compile("^([a-zA-Z]{2,5})-(.*)$");
     /**
      * Save skus to DB from mabang api.
      *
@@ -94,7 +97,7 @@ public class SkuListMabangServiceImpl extends ServiceImpl<SkuListMabangMapper, S
 
         /* for new skuDatas, insert them to DB */
         try {
-            if (newSkuDatas.size() != 0) {
+            if (!newSkuDatas.isEmpty()) {
                 // we need to check if the product associated with the sku exists, for that we are going to parse the Sku erpCode into product code
                 // check if the product code exists in DB, if not we create a new entry in DB and fill all the infos.
                 // then we can finally add the new Sku, product has to be created first if it doesn't exist, since we need to fill productID in Sku table
@@ -160,6 +163,63 @@ public class SkuListMabangServiceImpl extends ServiceImpl<SkuListMabangMapper, S
         return newSkusMap;
     }
 
+    @Override
+    @Transactional
+    public void updateSkusFromMabang(List<SkuData> skuDataList) {
+        if (skuDataList.isEmpty()) {
+            return;
+        }
+        // we collect all erpCode
+        List<String> allSkuErpCode = skuDataList.stream()
+                .map(SkuData::getErpCode)
+                .collect(toList());
+        // find Skus that already exist in DB
+        List<Sku> existingSkuList = skuListMabangMapper.searchExistence(allSkuErpCode);
+        // We map all existing Skus in DB with erpCode as key
+        Map<String, Sku> existingSkusIDMap = existingSkuList.stream()
+                .collect(
+                        Collectors.toMap(
+                                Sku::getErpCode, Function.identity()
+                        )
+                );
+
+        ArrayList<SkuData> existingSkuDatas = new ArrayList<>();
+        for (SkuData retrievedSkuData : skuDataList) {
+            Sku skuInDatabase = existingSkusIDMap.get(retrievedSkuData.getErpCode());
+            // the current SkuData's erpCode is in DB, so we add it to the list of existingSkuDatas
+            if (skuInDatabase != null) {
+                existingSkuDatas.add(retrievedSkuData);
+            }
+        }
+
+        /* for skuDatas to update, update product names and sku status them to DB */
+        try {
+            if (!existingSkuDatas.isEmpty()) {
+                // we need to check if the product associated with the sku exists, for that we are going to parse the Sku erpCode into product code
+                // check if the product code exists in DB, if not we create a new entry in DB and fill all the infos.
+                // then we can finally add the new Sku, product has to be created first if it doesn't exist, since we need to fill productID in Sku table
+                // we can now proceed to create new sku_declare_value associated with the new Sku and also sku_price
+                updateProductFromMabang(existingSkuDatas);
+                log.info("{} skus to be updated.", existingSkuDatas.size());
+
+                //update status of existing skus
+                List<Sku> skusToUpdate = new ArrayList<>();
+                for(SkuData skuData: existingSkuDatas) {
+                    Sku s = new Sku();
+                    s.setId(existingSkusIDMap.get(skuData.getErpCode()).getId());
+                    s.setUpdateBy("mabang api");
+                    s.setUpdateTime(new Date());
+                    s.setStatus(skuData.getStatusValue());
+                    skusToUpdate.add(s);
+                }
+                skuService.updateBatchById(skusToUpdate);
+                log.info("Updated {} skus : {}.", skusToUpdate.size(), existingSkuDatas.stream().map(SkuData::getErpCode).collect(toList()));
+            }
+        } catch (RuntimeException e) {
+            log.error(e.getLocalizedMessage());
+        }
+    }
+
     /**
      * Save products to DB from mabang api.
      *
@@ -197,6 +257,63 @@ public class SkuListMabangServiceImpl extends ServiceImpl<SkuListMabangMapper, S
         }
         return productsWithInfoMap;
     }
+    /**
+     * Update products enName and zhName to DB from mabang api.
+     *
+     * @param skuDataList we update the product enName and zhName of the skus
+     */
+    public void updateProductFromMabang(List<SkuData> skuDataList) {
+        List<String> allProductCodes = parseSkuListToProductCodeList(skuDataList);
+
+        List< Product> existingProduct = skuListMabangMapper.searchProductExistence(allProductCodes);
+        Map<String, Product> existingProductsIDMap = existingProduct.stream()
+                .collect(
+                        Collectors.toMap(
+                                Product::getCode, Function.identity()
+                        )
+                );
+        List<SkuData> skuDatasToUpdate = new ArrayList<>();
+        // we ignore the new products and only update the existing ones
+        for(SkuData skuData : skuDataList) {
+            Product productInDB = existingProductsIDMap.get(parseSkuToProduct(skuData.getErpCode()));
+            // the current product code is in DB, so we add it to the list of newProducts
+            if (productInDB != null) {
+                skuDatasToUpdate.add(skuData);
+            }
+        }
+        List<Product> productsToUpdate = new ArrayList<>();
+        for(SkuData skuData: skuDatasToUpdate) {
+            Product p = new Product();
+            p.setId(existingProductsIDMap.get(parseSkuToProduct(skuData.getErpCode())).getId());
+            p.setUpdateBy("mabang api");
+            p.setUpdateTime(new Date());
+            // Removing the customer code from the product CN name
+            if (!skuData.getNameEN().isEmpty()) {
+                Matcher enNameMatcher = enNamePattern.matcher(skuData.getNameEN());
+                if (enNameMatcher.matches() && !enNameMatcher.group(2).isEmpty()) {
+                    p.setEnName(enNameMatcher.group(2));
+                }
+                else {
+                    p.setEnName(skuData.getNameEN());
+                }
+            }
+            // Removing the customer code from the product CN name
+            if (!skuData.getNameCN().isEmpty()) {
+                Matcher cnNameMatcher = cnNamePattern.matcher(skuData.getNameCN());
+                if (cnNameMatcher.matches() && !cnNameMatcher.group(2).isEmpty()) {
+                    p.setZhName(cnNameMatcher.group(2));
+                }
+                else {
+                    p.setZhName(skuData.getNameCN());
+                }
+            }
+            productsToUpdate.add(p);
+        }
+        if(!productsToUpdate.isEmpty()) {
+            productService.updateBatchById(productsToUpdate);
+        }
+        log.info("Updated {} products : {}.", productsToUpdate.size(), skuDatasToUpdate.stream().map(SkuData::getErpCode).collect(toList()));
+    }
 
     public void saveSkuPrices(List<SkuData> newSkus) {
         List<SkuPrice> l = new ArrayList<>();
@@ -290,8 +407,6 @@ public class SkuListMabangServiceImpl extends ServiceImpl<SkuListMabangMapper, S
         final String electroMagSensitiveAttributeId = skuListMabangMapper.searchSensitiveAttributeId("Electro-magnetic");
         final String electricSensitiveAttributeId = skuListMabangMapper.searchSensitiveAttributeId("Electronic/Electric");
         final String normalSensitiveAttributeId = skuListMabangMapper.searchSensitiveAttributeId("Normal goods");
-        // In NameCN field on top of the product name we also get the customer code in the beginning of the string : "XX Description of the product"
-        final Pattern cnNamePattern = Pattern.compile("^([a-zA-Z]{2,5})\\s(.*)$");
         // IN saleRemark sometimes not only the product weight provided, we can get extra information such as service_fee (eg : "15每件服务费0.2")
         final Pattern saleRemarkPattern = Pattern.compile("^([0-9]*)(.*)$");
         // we are stocking product codes, so we don't get duplicates which causes error