From 6efc844067ed30210f217d2b169db19ab35e7476 Mon Sep 17 00:00:00 2001 From: "pref_mail@163.com" Date: Thu, 27 Feb 2025 16:15:07 +0800 Subject: [PATCH] =?UTF-8?q?add:=E5=A2=9E=E5=8A=A0=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E7=B4=A2=E8=8F=B2=E7=9A=84=E6=95=B0=E6=8D=AE=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=92=8C=E6=95=B0=E6=8D=AE=E6=8E=A8=E9=80=81=E5=88=B0=E5=AF=B9?= =?UTF-8?q?=E5=BA=94=E7=A7=9F=E6=88=B7=EF=BC=8C=E9=9C=80=E8=A6=81=E5=95=86?= =?UTF-8?q?=E5=AE=B6=E8=AE=BE=E5=AE=9A=E7=9A=84=E4=BB=93=E5=BA=93=E5=85=A5?= =?UTF-8?q?=E5=8E=82=E7=BA=BF=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/component/MockLoginService.java | 6 + .../aspect/JobAnnotationAspect.java | 19 + .../base/feign/FactoryDataBaseClient.java | 1 - .../suofeiya/config/ExecutorConfig.java | 134 +++++- .../suofeiya/entity/SpiderInforEntity.java | 36 ++ .../factorydata/suofeiya/job/ChdbTask.java | 25 ++ .../suofeiya/job/SpiderDataJob.java | 383 ++++++++++++------ .../suofeiya/mapper/SpiderInforMapper.java | 12 + .../suofeiya/mapper/SpiderInforMapper.xml | 5 + .../suofeiya/service/IAsyncService.java | 9 + .../service/impl/AsyncServiceImpl.java | 22 + .../service/impl/DeliveryNoteServiceImpl.java | 3 +- 12 files changed, 528 insertions(+), 127 deletions(-) create mode 100644 blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/entity/SpiderInforEntity.java create mode 100644 blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/ChdbTask.java create mode 100644 blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.java create mode 100644 blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.xml create mode 100644 blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/IAsyncService.java create mode 100644 blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/AsyncServiceImpl.java diff --git a/blade-biz-common/src/main/java/org/springblade/common/component/MockLoginService.java b/blade-biz-common/src/main/java/org/springblade/common/component/MockLoginService.java index 36e271271..e6af0bc57 100644 --- a/blade-biz-common/src/main/java/org/springblade/common/component/MockLoginService.java +++ b/blade-biz-common/src/main/java/org/springblade/common/component/MockLoginService.java @@ -19,6 +19,9 @@ import java.util.Objects; public class MockLoginService { private final Environment environment; public JSONObject mockToken(String tenantId,String account) { + + + log.info("mockToken >>> tenantId={} account={} ",tenantId,account); String url = "http://" + LauncherConstant.loginAddr(Objects.requireNonNull(environment.getActiveProfiles()[0])) + "/blade-auth/oauth/token"; HttpRequest urlRequest = HttpRequest.post(url); @@ -37,4 +40,7 @@ public class MockLoginService { return jsonObject; } + + + } diff --git a/blade-service/logpm-distribution/src/main/java/com/logpm/distribution/aspect/JobAnnotationAspect.java b/blade-service/logpm-distribution/src/main/java/com/logpm/distribution/aspect/JobAnnotationAspect.java index 0e4d422ba..9bbaf1264 100644 --- a/blade-service/logpm-distribution/src/main/java/com/logpm/distribution/aspect/JobAnnotationAspect.java +++ b/blade-service/logpm-distribution/src/main/java/com/logpm/distribution/aspect/JobAnnotationAspect.java @@ -78,5 +78,24 @@ public class JobAnnotationAspect { return data; } + private JSONObject mockLogin(String account, String tenantId) throws InterruptedException { + + String key =CacheNames.LOCAL_SERVER_USER+tenantId+":"+account; + String lockKey =key+":lock"; + JSONObject data =bladeRedis.get(key); + if(Objects.isNull(data)){ + boolean flag = redisLockClient.tryLock(lockKey, LockType.FAIR, 5000, 10000, TimeUnit.MILLISECONDS); + if(flag){ + data =bladeRedis.get(key); + if(Objects.isNull(data)){ + data = mockLoginService.mockToken(tenantId,account); + bladeRedis.setEx(key,data, DefaultUserTokenConfig.EXPIRE_TIME); + redisLockClient.unLock(lockKey, LockType.FAIR); + } + } + } + return data; + } + } diff --git a/blade-service/logpm-factory-data/logpm-factory-data-base/src/main/java/com/logpm/factorydata/base/feign/FactoryDataBaseClient.java b/blade-service/logpm-factory-data/logpm-factory-data-base/src/main/java/com/logpm/factorydata/base/feign/FactoryDataBaseClient.java index b9e3c7531..f9b9c4136 100644 --- a/blade-service/logpm-factory-data/logpm-factory-data-base/src/main/java/com/logpm/factorydata/base/feign/FactoryDataBaseClient.java +++ b/blade-service/logpm-factory-data/logpm-factory-data-base/src/main/java/com/logpm/factorydata/base/feign/FactoryDataBaseClient.java @@ -44,7 +44,6 @@ public class FactoryDataBaseClient implements IFactoryDataBaseClient { LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); queryWrapper.eq(FactoryWarehouseBindEntity::getBrand, brand); queryWrapper.eq(FactoryWarehouseBindEntity::getSiteName, siteName); - String tenantId = AuthUtil.getTenantId(); List warehouseList = bindService.list(queryWrapper); if (CollUtil.isNotEmpty(warehouseList)) { FactoryWarehouseBindEntity entity = warehouseList.get(0); diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/config/ExecutorConfig.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/config/ExecutorConfig.java index 247cff77b..2328d5519 100644 --- a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/config/ExecutorConfig.java +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/config/ExecutorConfig.java @@ -1,22 +1,35 @@ package com.logpm.factorydata.suofeiya.config; +import com.alibaba.fastjson.JSONObject; import com.baomidou.dynamic.datasource.toolkit.DynamicDataSourceContextHolder; +import com.logpm.factorydata.suofeiya.job.ChdbTask; +import com.logpm.factorydata.suofeiya.service.impl.AsyncServiceImpl; import lombok.extern.slf4j.Slf4j; import org.slf4j.MDC; +import org.springblade.common.cache.CacheNames; +import org.springblade.common.component.MockLoginService; +import org.springblade.common.utils.DefaultUserTokenConfig; +import org.springblade.core.redis.cache.BladeRedis; +import org.springblade.core.redis.lock.LockType; +import org.springblade.core.redis.lock.RedisLockClient; import org.springblade.core.secure.utils.AuthUtil; +import org.springblade.core.tool.utils.SpringUtil; import org.springblade.core.tool.utils.ThreadLocalUtil; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.task.TaskDecorator; +import org.springframework.http.HttpHeaders; +import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.web.context.request.RequestAttributes; import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; import javax.annotation.Nonnull; import java.util.Map; -import java.util.concurrent.Executor; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.Objects; +import java.util.concurrent.*; @Configuration @Slf4j @@ -65,7 +78,7 @@ public class ExecutorConfig { } RequestContextHolder.setRequestAttributes(context); // 未当前的异步线程绑定租户ID 和切换数据源 - log.info(">>>> 异步线程创建，绑定租户数据源 {}",tenantId); + log.info(">>>> 异步线程创建，绑定租户数据源 {}", tenantId); DynamicDataSourceContextHolder.push(tenantId); runnable.run(); } finally { @@ -82,4 +95,119 @@ public class ExecutorConfig { } } + + @Bean + public Executor asyncChangeDbExecutor() { + log.info("start async executor"); + ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor(); +// 配置核心线程数 + threadPoolTaskExecutor.setCorePoolSize(10); +// 配置最大线程数 + threadPoolTaskExecutor.setMaxPoolSize(20); +// 配置队列大小 + threadPoolTaskExecutor.setQueueCapacity(50); +// 配置线程池中线程的名称前缀 + threadPoolTaskExecutor.setThreadNamePrefix("ASYNC_DB_THREAD_"); +// HelloWorldServiceImpl rejection-policy: 当pool已经达到max size时，如何处理新任务： +// CallerRunsPolicy: 不在新线程中执行任务，而是由调用者所在的线程来执行； +// AbortPolicy: 拒绝执行新任务，并抛出RejectedExecutionException异常； +// DiscardPolicy：丢弃当前将要加入队列的任务； +// DiscardOldestPolicy：丢弃任务队列中最旧的任务； + threadPoolTaskExecutor.setRejectedExecutionHandler( + new ThreadPoolExecutor.CallerRunsPolicy() + ); + threadPoolTaskExecutor.setTaskDecorator(new ContextDBCopyingDecorator()); + threadPoolTaskExecutor.initialize(); + return threadPoolTaskExecutor; + } + + + static class ContextDBCopyingDecorator implements TaskDecorator { + @Nonnull + @Override + public Runnable decorate(@Nonnull Runnable runnable) { + RequestAttributes context = RequestContextHolder.currentRequestAttributes(); + + if (runnable instanceof ChdbTask) { + + + ChdbTask myTask = (ChdbTask) runnable; + String tenantId = myTask.getTenantId(); + + + JSONObject jsonObject = null; + try { + jsonObject = mockLogin(tenantId); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + MockHttpServletRequest mockRequest = new MockHttpServletRequest(); + mockRequest.addHeader("Blade-Auth", "bearer "+jsonObject.get("access_token")); + RequestContextHolder.setRequestAttributes(new ServletRequestAttributes(mockRequest)); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.add("Blade-Auth","bearer "+jsonObject.get("access_token") ); + httpHeaders.add( "Authorization", "Basic bG9jYWw6bG9jYWxfc2VjcmV0"); + ThreadLocalUtil.put("bladeContext", httpHeaders); + + Map all = ThreadLocalUtil.getAll(); + Map mdcMap = MDC.getCopyOfContextMap(); + return () -> { + try { + all.keySet().forEach(key -> ThreadLocalUtil.put(key, all.get(key))); + if (mdcMap != null && !mdcMap.isEmpty()) { + MDC.setContextMap(mdcMap); + } + RequestContextHolder.setRequestAttributes(context); + // 未当前的异步线程绑定租户ID 和切换数据源 + log.info(">>>> ContextDBCopyingDecorator 异步线程创建，绑定租户数据源 {}", tenantId); + DynamicDataSourceContextHolder.push(tenantId); + runnable.run(); + } finally { + RequestContextHolder.resetRequestAttributes(); + all.clear(); + if (mdcMap != null) { + mdcMap.clear(); + } + ThreadLocalUtil.clear(); + MDC.clear(); + DynamicDataSourceContextHolder.poll(); + } + }; + } + return null; + + } + + private JSONObject mockLogin(String tenantId) throws InterruptedException { + + // 获取Spring 容器中的 + BladeRedis bladeRedis= SpringUtil.getBean(BladeRedis.class); + MockLoginService mockLoginService = SpringUtil.getBean(MockLoginService.class); + RedisLockClient redisLockClient = SpringUtil.getBean(RedisLockClient.class); + String account ="shujutongbu"; + + String key = CacheNames.LOCAL_SERVER_USER+tenantId+":"+account; + String lockKey =key+":lock"; + JSONObject data =bladeRedis.get(key); + if(Objects.isNull(data)){ + boolean flag = redisLockClient.tryLock(lockKey, LockType.FAIR, 5000, 10000, TimeUnit.MILLISECONDS); + if(flag){ + data =bladeRedis.get(key); + if(Objects.isNull(data)){ + data = mockLoginService.mockToken(tenantId,account); + bladeRedis.setEx(key,data, DefaultUserTokenConfig.EXPIRE_TIME); + redisLockClient.unLock(lockKey, LockType.FAIR); + } + } + } + return data; + + + + + } + + + } } diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/entity/SpiderInforEntity.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/entity/SpiderInforEntity.java new file mode 100644 index 000000000..132383752 --- /dev/null +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/entity/SpiderInforEntity.java @@ -0,0 +1,36 @@ +package com.logpm.factorydata.suofeiya.entity; + +import com.baomidou.mybatisplus.annotation.TableName; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.springblade.core.mp.base.BaseEntity; + +@Data +@TableName("sfy_spider_infor") +@ApiModel(value = "装箱包信息", description = "装箱包信息") +@EqualsAndHashCode(callSuper = true) +public class SpiderInforEntity extends BaseEntity { + + @ApiModelProperty(name = "登录账号", notes = "") + private String userAccount; + + + @ApiModelProperty(name = "登录密码", notes = "") + private String userPasswd; + + + @ApiModelProperty(name = "绑定的商家租户号", notes = "") + private String bindTenantCode; + + + @ApiModelProperty(name = "绑定商家名称", notes = "") + private String bindBussiness; + + @ApiModelProperty(name = "物流编码-- 用于识别商场绑定的仓库", notes = "") + private String carrierNo; + + + +} diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/ChdbTask.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/ChdbTask.java new file mode 100644 index 000000000..80f2375c8 --- /dev/null +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/ChdbTask.java @@ -0,0 +1,25 @@ +package com.logpm.factorydata.suofeiya.job; + +import com.logpm.factorydata.suofeiya.service.DeliveryNoteService; +import com.logpm.factorydata.suofeiya.vo.DeliveryNoteVO; +import lombok.Data; + +@Data +public class ChdbTask implements Runnable { + private String tenantId; + + private DeliveryNoteVO vo; + + private DeliveryNoteService deliveryNoteService; + + public ChdbTask(String tenantId, DeliveryNoteVO vo, DeliveryNoteService deliveryNoteService) { + this.tenantId = tenantId; + this.vo = vo; + this.deliveryNoteService = deliveryNoteService; + } + + @Override + public void run() { + deliveryNoteService.buildAdvance(vo); + } +} diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/SpiderDataJob.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/SpiderDataJob.java index d12d75785..3526cc1aa 100644 --- a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/SpiderDataJob.java +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/job/SpiderDataJob.java @@ -7,24 +7,64 @@ import cn.hutool.http.HttpUtil; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.logpm.factorydata.suofeiya.entity.DeliveryNoteEntity; +import com.logpm.factorydata.suofeiya.entity.OrderInfoEntity; +import com.logpm.factorydata.suofeiya.entity.PackageInfoEntity; +import com.logpm.factorydata.suofeiya.entity.SpiderInforEntity; +import com.logpm.factorydata.suofeiya.mapper.DeliveryNoteMapper; +import com.logpm.factorydata.suofeiya.mapper.OrderInfoMapper; +import com.logpm.factorydata.suofeiya.mapper.PackageInfoMapper; +import com.logpm.factorydata.suofeiya.mapper.SpiderInforMapper; +import com.logpm.factorydata.suofeiya.service.DeliveryNoteService; +import com.logpm.factorydata.suofeiya.service.IAsyncService; +import com.logpm.factorydata.suofeiya.vo.DeliveryNoteVO; +import com.logpm.factorydata.suofeiya.vo.OrderInfoVO; +import com.logpm.factorydata.suofeiya.vo.PackageInfoVO; import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.annotation.XxlJob; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springblade.core.tool.utils.BeanUtil; import org.springblade.core.tool.utils.StringUtil; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.net.HttpCookie; import java.time.LocalDate; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; @Component @Slf4j public class SpiderDataJob { - private Map map = new HashMap<>(); + private Map map = new HashMap<>(); + + private String startData = "2025-02-18"; + private String endData = "2025-02-19"; + + @Autowired + private SpiderInforMapper spiderInforMapper; + + @Autowired + private DeliveryNoteMapper deliveryNoteMapper; + @Autowired + private OrderInfoMapper orderInfoMapper; + + @Autowired + private PackageInfoMapper packageInfoMapper; + @Autowired + private DeliveryNoteService deliveryNoteService; + + @Autowired + private Executor asyncChangeDbExecutor; + + + /** @@ -37,55 +77,118 @@ public class SpiderDataJob { @XxlJob("factoryCarsData") public ReturnT factoryCarsData(String param) throws Exception { - -// String date =LocalDate.now().toString(); + startData =LocalDate.now().toString(); + endData = LocalDate.now().minusDays(1).toString(); // -// String yesterday = LocalDate.now().minusDays(1).toString(); - - -// // 获取当前日期 -// String date = DateUtil.today(); -// // 获取当前日期的前一天的日期 -// String yesterday = DateUtil.offsetDay(DateUtil.parse(date), -1) - - - Long _dc_ = System.currentTimeMillis(); - - String url = "https://portal.sfygroup.com/reporthome/rest/loadingHeader/all?_dc=" + _dc_ + "&page=1&start=0&limit=100&" + - "filtersRaw=[{\"id\":null,\"property\":\"A.DEAL_DATE\",\"value\":\"" + "2025-01-18" + "\",\"operator\":\">=\",\"sql\":null}," + - "{\"id\":null,\"property\":\"A.DEAL_DATE\",\"value\":\""+"2025-01-19"+"\",\"operator\":\"<\",\"sql\":null}]"; - System.out.println(url); - - Map heads = new HashMap<>(); - heads.put("Content-Type", "application/json;charset=UTF-8"); - heads.put("Cookie", buildCookies()); - HttpRequest get = HttpUtil.createGet(url).addHeaders(heads); - HttpResponse execute = get.setReadTimeout(10000).execute(); - String body = execute.body(); +// startData ="2025-02-25"; +// endData = "2025-02-27"; + + + // 获取需要处理的账号表 + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", 1); + List list = spiderInforMapper.selectList(queryWrapper); + + for (SpiderInforEntity spiderInforEntity : list) { + // 读取获取的账号 + Long _dc_ = System.currentTimeMillis(); + + String url = "https://portal.sfygroup.com/reporthome/rest/loadingHeader/all?_dc=" + _dc_ + "&page=1&start=0&limit=100&" + + "filtersRaw=[{\"id\":null,\"property\":\"A.DEAL_DATE\",\"value\":\"" + startData + "\",\"operator\":\">=\",\"sql\":null}," + + "{\"id\":null,\"property\":\"A.DEAL_DATE\",\"value\":\"" + endData + "\",\"operator\":\"<\",\"sql\":null}]"; + + JSONObject jsonObject = getData(url, buildCookies(spiderInforEntity.getUserAccount(), spiderInforEntity.getUserPasswd())); + JSONArray jsonArray = jsonObject.getJSONArray("result"); + + for (Object o : jsonArray) { + JSONObject jsonObject1 = (JSONObject) o; + DeliveryNoteVO vo = new DeliveryNoteVO(); + DeliveryNoteEntity deliveryNoteEntity = saveDeliveyData(spiderInforEntity, jsonObject1); + + BeanUtil.copy(deliveryNoteEntity, vo); + vo.setCarrierNo(spiderInforEntity.getCarrierNo()); + + if(deliveryNoteEntity != null){ + // 得到车次下的订单 + List orderInfo =new ArrayList<>(); + List packageInfoVOs =new ArrayList<>(); + JSONArray orders = getCarOrders(jsonObject1.getStr("deliveryHeaderId"), spiderInforEntity.getUserAccount(), spiderInforEntity.getUserPasswd()); + for (Object o1 : orders) { + JSONObject jsonObject2 = (JSONObject) o1; + // 保存订单数据 + String orderNumber = jsonObject2.getStr("orderNumber"); + + String customName = jsonObject2.getStr("customName"); + + OrderInfoEntity orderInfoEntity = new OrderInfoEntity(); + orderInfoEntity.setOrderNo(orderNumber); + orderInfoEntity.setOrderFull(orderNumber); + orderInfoEntity.setReceiver(customName); + orderInfoEntity.setConnectName(customName); + orderInfoEntity.setTenantCode(spiderInforEntity.getBindTenantCode()); + orderInfoEntity.setDeliveryNoteId(deliveryNoteEntity.getId().toString()); + orderInfoEntity.setCreateTime(new Date()); + orderInfoEntity.setUpdateTime(new Date()); + orderInfoEntity.setIsDeleted(0); + orderInfoEntity.setStatus(1); + + // 判断订单在数据库是否已经存在 如果不存在 插入到数据库中 + if(orderInfoMapper.selectCount(new QueryWrapper().eq("order_no", orderNumber)) == 0){ + orderInfoMapper.insert(orderInfoEntity); + OrderInfoVO orderInfoVo = new OrderInfoVO(); + BeanUtil.copy(orderInfoEntity, orderInfoVo); + orderInfo.add(orderInfoVo); + } + + + // 得到订单下的包条信息 + JSONArray orderInfor = getOrderPackages(orderNumber, spiderInforEntity.getUserAccount(), spiderInforEntity.getUserPasswd()); + + for (Object object : orderInfor) { + JSONObject jsonObject3 = (JSONObject) object; + + String packageNumber = jsonObject3.getStr("tagNumber"); + String packType = jsonObject3.getStr("packType"); + String volume = jsonObject3.getStr("volume"); + String weight = jsonObject3.getStr("weight"); + + PackageInfoEntity packageInfoEntity = new PackageInfoEntity(); + packageInfoEntity.setDeliveryNoteId(deliveryNoteEntity.getId().toString()); + packageInfoEntity.setPaNo(packageNumber); + packageInfoEntity.setOrderType(jsonObject1.getStr("orderType")); + packageInfoEntity.setPlDepart(packType); + packageInfoEntity.setVolume(volume); + packageInfoEntity.setWeight(weight); + packageInfoEntity.setTenantCode(spiderInforEntity.getBindTenantCode()); + packageInfoEntity.setOrderNo(orderNumber); + packageInfoEntity.setCreateTime(new Date()); + packageInfoEntity.setUpdateTime(new Date()); + packageInfoEntity.setIsDeleted(0); + packageInfoEntity.setStatus(1); + + // 判断这个包条是否存在鱼数据库 如果不存在 则执行 + if(StringUtil.isNotBlank(packageNumber) &&packageInfoMapper.selectCount(new QueryWrapper().eq("pa_no", packageNumber)) == 0){ + packageInfoMapper.insert(packageInfoEntity); + + PackageInfoVO packageInfoVO = new PackageInfoVO(); + BeanUtil.copy(packageInfoEntity, packageInfoVO); + + packageInfoVOs.add(packageInfoVO); + + } + } + } + vo.setOrderInfo(orderInfo); + vo.setPackageInfo(packageInfoVOs); + } - JSONObject jsonObject = new JSONObject(body); - JSONArray jsonArray = jsonObject.getJSONArray("result"); - - for (Object o : jsonArray) { - JSONObject jsonObject1 = (JSONObject) o; - System.out.println("车次"+jsonObject1); - // 得到车次下的订单 - JSONArray orders = getCarOrders(jsonObject1.getStr("deliveryHeaderId")); - System.out.println("订单"+orders); -// // 得到订单下的包条信息 -// for (Object o1 : orders) { -// JSONObject jsonObject2 = (JSONObject) o1; -// System.out.println("订单"+jsonObject2); -// // 得到订单下的包条信息 -// JSONArray orderInfor = getOrderPackages(jsonObject2.getStr("orderNumber")); -// System.out.println("包件"+orderInfor); -// } + asyncChangeDbExecutor.execute(new ChdbTask(spiderInforEntity.getBindTenantCode(), vo,deliveryNoteService)); + } } - /** * { * "total": 10000, @@ -192,43 +295,32 @@ public class SpiderDataJob { } - - public static void main(String[] args) throws Exception { - - - SpiderDataJob s = new SpiderDataJob(); - s.factoryCarsData(""); - - -// Long s = System.currentTimeMillis(); -// System.out.println(s); -// System.out.println(s / 1000); - - -// String url = "https://portal.sfygroup.com/reporthome/rest/loadingHeader/all?_dc=1739871711195&page=1&start=0&limit=100&filtersRaw=[{\"id\":null,\"property\":\"A.DEAL_DATE\",\"value\":\"2025-02-11\",\"operator\":\">=\",\"sql\":null},{\"id\":null,\"property\":\"A.DEAL_DATE\",\"value\":\"2025-02-19\",\"operator\":\"<\",\"sql\":null}]"; -//// // 发起http请求 获取返回的数据 走get请求 -//// -//// -//// -// Map heads = new HashMap<>(); -// heads.put("Content-Type", "application/json;charset=UTF-8"); -//// heads.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"); -//// heads.put("Host", "portal.sfygroup.com"); -//// heads.put("Accept", "*/*"); -// -// heads.put("Cookie", init()); -//// heads.put("Referer", "http://portal.sfygroup.com/reporthome/rest/reporthome/reporthomeUI?username=18166360906&password=02a962f8b9dfdbe85ee37b3f9b99fd8fcb48e73a65c9009e92ab81c508dcdca261af7efc1644002d0f371e5001c30a8f&menuType=loadingheader/loadingheader.jsp"); -// HttpRequest get = HttpUtil.createGet(url).addHeaders(heads); -//// String s = HttpUtil.get(url); -//// System.out.println(s); -// -// HttpResponse execute = get.setReadTimeout(10000).execute(); -// String body = execute.body(); -// System.out.println(body); - - -// init(); - + private DeliveryNoteEntity saveDeliveyData(SpiderInforEntity spiderInforEntity, JSONObject jsonObject1) { + // 获取车次数据 + // 装车单号 + String deliveryNumber = jsonObject1.getStr("deliveryNumber"); + // 车牌号 + String customerNumber = jsonObject1.getStr("customerNumber"); + + DeliveryNoteEntity deliveryNoteEntity = new DeliveryNoteEntity(); + deliveryNoteEntity.setLogisticID(deliveryNumber); + deliveryNoteEntity.setTruckNo(customerNumber); + deliveryNoteEntity.setTenantCode(spiderInforEntity.getBindTenantCode()); + deliveryNoteEntity.setSupplierName(spiderInforEntity.getBindBussiness()); + deliveryNoteEntity.setCreateTime(new Date()); + deliveryNoteEntity.setUpdateTime(new Date()); + deliveryNoteEntity.setIsDeleted(0); + deliveryNoteEntity.setStatus(1); + + + // 判断车次数据是否在表已存在 + DeliveryNoteEntity deliveryNoteEntity1 = deliveryNoteMapper.selectOne(new QueryWrapper().eq("logistic_id", deliveryNumber).eq("tenant_code", spiderInforEntity.getBindTenantCode())); + if (deliveryNoteEntity1 == null) { + // 不存在 则插入数据库 + deliveryNoteMapper.insert(deliveryNoteEntity); + return deliveryNoteEntity; + } + return null; } @@ -237,22 +329,51 @@ public class SpiderDataJob { * * @return */ - private String buildCookies() { - String o = map.get("cookie"); - if(StringUtil.isBlank(o)){ - String url = "https://portal.sfygroup.com/reporthome/rest/reporthome/reporthomeUI?username=18166360906&password=02a962f8b9dfdbe85ee37b3f9b99fd8fcb48e73a65c9009e92ab81c508dcdca261af7efc1644002d0f371e5001c30a8f&menuType=loadingheader/loadingheader.jsp"; + private String buildCookies(String username, String password) { + String o = map.get("cookie"); + if (StringUtil.isBlank(o)) { + String url = "https://portal.sfygroup.com/reporthome/rest/reporthome/reporthomeUI?username=" + username + "&password=" + password + "&menuType=loadingheader/loadingheader.jsp"; HttpRequest get = HttpUtil.createGet(url); HttpResponse execute = get.execute(); String cookie = execute.getCookieStr(); map.put("cookie", cookie); - o=cookie; + o = cookie; + } + + return o; + } + + + private String buildCookiesByZx(String username, String password) { + String o = map.get("Zx_cookie"); + if (StringUtil.isBlank(o)) { + String url = "https://portal.sfygroup.com/reporthome/rest/reporthome/reporthomeUI?username=" + username + "&password=" + password + "&menuType=orderpack/OrderPack.jsp"; + HttpRequest get = HttpUtil.createGet(url); + HttpResponse execute = get.execute(); + String cookie = execute.getCookieStr(); + map.put("Zx_cookie", cookie); + o = cookie; } return o; + } + private String buildCookiesByDD(String username, String password) { + String o = map.get("DD_cookie"); + if (StringUtil.isBlank(o)) { + String url = "https://portal.sfygroup.com/reporthome/rest/reporthome/reporthomeUI?username=" + username + "&password=" + password + "&menuType=delivedryNotice/DelivedryNotice.jsp"; + HttpRequest get = HttpUtil.createGet(url); + HttpResponse execute = get.execute(); + String cookie = execute.getCookieStr(); + map.put("DD_cookie", cookie); + o = cookie; + } + + return o; } + /** * 通过车次获取下面的订单 * {"total":10000,"result": @@ -264,41 +385,56 @@ public class SpiderDataJob { * @param id * @return */ - private JSONArray getCarOrders(String id) { + private JSONArray getCarOrders(String id, String username, String password) { + + // 输出这个方法执行的时间 + long start =System.currentTimeMillis(); + JSONArray result = new JSONArray(); Long _dc_ = System.currentTimeMillis(); - String url = "https://portal.sfygroup.com/reporthome/rest/loadingLine/all?_dc=" + _dc_ + "&page=1&start=0&limit=10&filtersRaw=[{\"id\":\"headFilter\",\"property\":\"A.DELIVERY_HEADER_ID\",\"value\":\"" + id + "\",\"operator\":\"=\",\"sql\":null}]"; + String url = "https://portal.sfygroup.com/reporthome/rest/loadingLine/all?_dc=" + _dc_ + "&page=1&start=0&limit=1000&filtersRaw=[{\"id\":\"headFilter\",\"property\":\"A.DELIVERY_HEADER_ID\",\"value\":\"" + id + "\",\"operator\":\"=\",\"sql\":null}]"; - JSONObject jsonObject = getData(url); + JSONObject jsonObject = getData(url, buildCookies(username, password)); JSONArray temList = jsonObject.getJSONArray("result"); for (Object o : temList) { JSONObject jsonObject1 = (JSONObject) o; + JSONObject list2 = getOrderInfor(jsonObject1.getStr("orderNumber"), username, password); - result.add(getOrderInfor(jsonObject1.getStr("orderNumber"))); + JSONArray list_2 = list2.getJSONArray("result"); + for (Object o1 : list_2) { + JSONObject t = (JSONObject) o1; + result.add(t); + } } + log.info("getCarOrders 执行时间：{}", System.currentTimeMillis()-start); return result; } - private JSONObject getOrderInfor(String orderCode) { - Long _dc_ = System.currentTimeMillis(); - String url = "https://portal.sfygroup.com/portal/rest/sogalOrder/all?_dc="+_dc_+"&isSlave=true&methodName=orderQuery&maskPhone=true&page=1&start=0&limit=10&filtersRaw=[{\"id\":\"queryorderinfoFormorderNumberFilter\",\"property\":\"orderNumber\",\"value\":\""+orderCode+"\",\"operator\":\"=\",\"sql\":null}]"; -// JSONObject jsonObject= getData(url); - + /** + * {"total":10000,"result":[{"id":null,"createdBy":null, + * "created":null,"lastUpdated":null,"lastUpdatedBy":null,"rowVersion":0,"tableName":null,"orderNumber":"P3516012400366-1Y", + * "orderType":"备货单","scheduleDeliveryDate":"2025-02-18 00:00:00","depart":"川橱烤漆内委(1.0),川内委石材(1.0)", + * "weight":0.0,"volume":0.003,"quantity":2,"bfComments":null,"customerNumber":"P35160","sequences":12,"orderBrandCode":"1","orderBrand":"索菲亚品牌", + * "ydOrderNumber":"P3516012400366-1Y","customerName":"陈娟","productOrgName":"OU_成都索菲亚"}],"success":true} + * + * @param orderCode + * @param username + * @param password + * @return + */ - Map heads = new HashMap<>(); - heads.put("Content-Type", "application/json;charset=UTF-8"); - heads.put("Cookie", buildBussiesCookies()); - heads.put("Referer", "https://portal.sfygroup.com/portal/views/home.jsp"); - HttpRequest get = HttpUtil.createGet(url).addHeaders(heads); - HttpResponse execute = get.setReadTimeout(10000).execute(); - String body = execute.body(); - JSONObject jsonObject = JSONUtil.parseObj(body); + private JSONObject getOrderInfor(String orderCode, String username, String password) { + Long _dc_ = System.currentTimeMillis(); +// String url = "https://portal.sfygroup.com/reporthome/rest/delivedryNotice/all?_dc=" + _dc_ + "&page=1&start=0&limit=1000&filtersRaw=" + +// "[{\"id\":null,\"property\":\"A.ORDER_NUMBER\",\"value\":" + orderCode + ",\"operator\":\"=\",\"sql\":null}]"; + String url ="https://portal.sfygroup.com/reporthome/rest/delivedryNotice/all?_dc="+_dc_+"&page=1&start=0&limit=20&filtersRaw=[{\"id\":null,\"property\":\"A.SCHEDULE_DELIVERY_DATE\",\"value\":\""+startData+"\",\"operator\":\">=\",\"sql\":null},{\"id\":null,\"property\":\"A.SCHEDULE_DELIVERY_DATE\",\"value\":\""+endData+"\",\"operator\":\"<\",\"sql\":null},{\"id\":null,\"property\":\"A.ORDER_NUMBER\",\"value\":\""+orderCode+"\",\"operator\":\"like\",\"sql\":null}]"; + JSONObject jsonObject = getData(url, buildCookiesByDD(username, password)); return jsonObject; } @@ -321,16 +457,15 @@ public class SpiderDataJob { for (HttpCookie cookie : cookies) { if (cookie.getName().equals("JSESSIONID")) { - if(cookie.getValue().contains(".")){ - s=cookie.getValue().substring(0,cookie.getValue().indexOf(".")); + if (cookie.getValue().contains(".")) { + s = cookie.getValue().substring(0, cookie.getValue().indexOf(".")); } } } - - s="JSESSIONID="+s+";username=18166360906;JSESSIONID="+s; + s = "JSESSIONID=" + s + ";username=18166360906;JSESSIONID=" + s; map.put("buss_cookie", s); } @@ -338,24 +473,27 @@ public class SpiderDataJob { } - private JSONArray getOrderPackages(String orderCode) { + private JSONArray getOrderPackages(String orderCode, String username, String password) { JSONArray response = new JSONArray(); //获取订单对应的装箱单 Long _dc_ = System.currentTimeMillis(); - String url = "https://portal.sfygroup.com/reporthome/rest/orderPackHeader/all?_dc="+_dc_+"&page=1&start=0&limit=10000&filtersRaw=[{\"id\":\"headFilter\",\"property\":\"A.OE_HEADER_ID\",\"value\":\""+orderCode+"\",\"operator\":\"=\",\"sql\":null}]"; + String url = "https://portal.sfygroup.com/reporthome/rest/orderPackHead/all?_dc=" + _dc_ + "&page=1&start=0&limit=1000&filtersRaw=[{\"id\":null,\"property\":\"A.ORDER_NUMBER\",\"value\":\"" + orderCode + "\",\"operator\":\"=\",\"sql\":null},{\"id\":null,\"property\":\"A.SCHEDULE_DELIVERY_DATE\",\"value\":\"" + startData + "\",\"operator\":\">=\",\"sql\":null},{\"id\":null,\"property\":\"A.SCHEDULE_DELIVERY_DATE\",\"value\":\"" + endData + "\",\"operator\":\"<\",\"sql\":null}]"; /** * {"total":10000,"result":[{"id":null,"createdBy":null,"created":null,"lastUpdated":null,"lastUpdatedBy":null,"rowVersion":0,"tableName":"SG_ORG_BALANCE","orderNumber":"P3516012400306-60", * "sourceOrderNumber":"P3516012400306-60","orderType":"CD_外部图纸单","deliveryDate":"2025-02-18 00:00:00", * "quotationDate":"2024-12-27 12:14:26","oeHeaderId":45757470,"organizationId":"141","customerNumber":"P351688","orderBrandCode":"1","orderBrand":"索菲亚品牌"}],"success":true} */ - JSONObject data = getData(url); + JSONObject data = getData(url, buildCookiesByZx(username, password)); JSONArray tempList = data.getJSONArray("result"); for (Object o : tempList) { JSONObject jsonObject = (JSONObject) o; String oeHeaderId = jsonObject.getStr("oeHeaderId"); - url = "https://portal.sfygroup.com/reporthome/rest/orderPackLine/all?_dc="+_dc_+"&page=1&start=0&limit=10000&filtersRaw=[{\"id\":\"headFilter\",\"property\":\"A.OE_HEADER_ID\",\"value\":"+oeHeaderId+",\"operator\":\"=\",\"sql\":null}]"; - JSONObject data1 = getData(url); +// url = "https://portal.sfygroup.com/reporthome/rest/orderPackLine/all?_dc=" + _dc_ + "&page=1&start=0&limit=10000&filtersRaw=[{\"id\":\"headFilter\",\"property\":\"A.OE_HEADER_ID\",\"value\":" + oeHeaderId + ",\"operator\":\"=\",\"sql\":null}]"; + + url ="https://portal.sfygroup.com/reporthome/rest/orderPackLine/all?_dc="+_dc_+"&page=1&start=0&limit=10000&filtersRaw=[{\"id\":\"headFilter\",\"property\":\"A.OE_HEADER_ID\",\"value\":\""+oeHeaderId+"\",\"operator\":\"=\",\"sql\":null}]"; + + JSONObject data1 = getData(url, buildCookiesByZx(username, password)); /** * id null * createdBy null @@ -384,30 +522,31 @@ public class SpiderDataJob { */ // 获取列表 JSONArray result = data1.getJSONArray("result"); - response.addAll(result); + response.addAll(result); } return response; - - - - } - private JSONObject getData(String url) { + private JSONObject getData(String url, String cookie) { + Long start = System.currentTimeMillis(); Map heads = new HashMap<>(); heads.put("Content-Type", "application/json;charset=UTF-8"); - heads.put("Cookie", buildCookies()); + heads.put("Cookie", cookie); HttpRequest get = HttpUtil.createGet(url).addHeaders(heads); - HttpResponse execute = get.setReadTimeout(10000).execute(); + HttpResponse execute = get.setConnectionTimeout(5000).setReadTimeout(3000).execute(); String body = execute.body(); JSONObject jsonObject = JSONUtil.parseObj(body); + log.info("getData 执行时间：{}", System.currentTimeMillis()-start); + return jsonObject; } } + + diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.java new file mode 100644 index 000000000..eee54d21a --- /dev/null +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.java @@ -0,0 +1,12 @@ +package com.logpm.factorydata.suofeiya.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.logpm.factorydata.suofeiya.entity.SpiderInforEntity; +import org.apache.ibatis.annotations.Mapper; + +/** + * @description: 索菲亚爬虫信息表 + */ +@Mapper +public interface SpiderInforMapper extends BaseMapper { +} diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.xml b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.xml new file mode 100644 index 000000000..dc61d3e16 --- /dev/null +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/mapper/SpiderInforMapper.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/IAsyncService.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/IAsyncService.java new file mode 100644 index 000000000..9263690f3 --- /dev/null +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/IAsyncService.java @@ -0,0 +1,9 @@ +package com.logpm.factorydata.suofeiya.service; + +import com.logpm.factorydata.suofeiya.vo.DeliveryNoteVO; + +public interface IAsyncService { + + void buildAdvance(String tenantId, DeliveryNoteVO vo); + +} diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/AsyncServiceImpl.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/AsyncServiceImpl.java new file mode 100644 index 000000000..62d3e4ca5 --- /dev/null +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/AsyncServiceImpl.java @@ -0,0 +1,22 @@ +package com.logpm.factorydata.suofeiya.service.impl; + +import com.logpm.factorydata.suofeiya.service.DeliveryNoteService; +import com.logpm.factorydata.suofeiya.service.IAsyncService; +import com.logpm.factorydata.suofeiya.vo.DeliveryNoteVO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class AsyncServiceImpl implements IAsyncService { + + + @Async("asyncChangeDbExecutor") + @Override + public void buildAdvance(String tenantId, DeliveryNoteVO vo) { + + + } +} diff --git a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/DeliveryNoteServiceImpl.java b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/DeliveryNoteServiceImpl.java index 4fcf56d6d..0f421a4f3 100644 --- a/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/DeliveryNoteServiceImpl.java +++ b/blade-service/logpm-factory-data/logpm-factory-data-suofeiya/src/main/java/com/logpm/factorydata/suofeiya/service/impl/DeliveryNoteServiceImpl.java @@ -39,6 +39,7 @@ import org.springblade.core.secure.utils.AuthUtil; import org.springblade.core.tool.api.R; import org.springblade.resource.feign.IOssClient; import org.springframework.http.MediaType; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.commons.CommonsMultipartFile; @@ -96,7 +97,7 @@ public class DeliveryNoteServiceImpl extends BaseServiceImpl