feat: 增加入库原始日线信息

.idea/data_source_mapping.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourcePerFileMappings">
+    <file url="file://$APPLICATION_CONFIG_DIR$/consoles/db/a8b9cd0a-335e-42ae-991a-f2733200afbf/console.sql" value="a8b9cd0a-335e-42ae-991a-f2733200afbf" />
+  </component>
+</project>

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/LeopardServerApplication.java

@@ -1,6 +1,9 @@
 package com.lanyuanxiaoyao.leopard.server;
 
 import com.blinkfox.fenix.EnableFenix;
+import com.lanyuanxiaoyao.leopard.server.service.TuShareService;
+import com.yomahub.liteflow.core.FlowExecutor;
+import jakarta.annotation.Resource;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.boot.ApplicationArguments;
 import org.springframework.boot.ApplicationRunner;
@@ -21,7 +24,14 @@ public class LeopardServerApplication implements ApplicationRunner {
     SpringApplication.run(LeopardServerApplication.class, args);
   }
 
+  @Resource
+  private FlowExecutor executor;
+  @Resource
+  private TuShareService tuShareService;
+
   @Override
   public void run(ApplicationArguments args) {
+    executor.execute2RespWithEL("THEN(update_daily)");
+    // executor.execute2RespWithEL("THEN(update_stock)");
   }
 }

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/entity/Daily.java

@@ -0,0 +1,58 @@
+package com.lanyuanxiaoyao.leopard.server.entity;
+
+import com.lanyuanxiaoyao.leopard.server.Constants;
+import com.lanyuanxiaoyao.service.template.entity.SimpleEntity;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.EntityListeners;
+import jakarta.persistence.ManyToOne;
+import jakarta.persistence.Table;
+import java.time.LocalDate;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.FieldNameConstants;
+import org.hibernate.annotations.Comment;
+import org.hibernate.annotations.DynamicInsert;
+import org.hibernate.annotations.DynamicUpdate;
+import org.hibernate.annotations.SoftDelete;
+import org.springframework.data.jpa.domain.support.AuditingEntityListener;
+
+@Setter
+@Getter
+@ToString(callSuper = true)
+@FieldNameConstants
+@Entity
+@SoftDelete
+@DynamicUpdate
+@DynamicInsert
+@EntityListeners(AuditingEntityListener.class)
+@Table(name = Constants.DATABASE_PREFIX + "daily")
+public class Daily extends SimpleEntity {
+  @Column(nullable = false)
+  private LocalDate tradeDate;
+  @Comment("开盘价")
+  private Double open;
+  @Comment("最高价")
+  private Double high;
+  @Comment("最低价")
+  private Double low;
+  @Comment("收盘价")
+  private Double close;
+  @Comment("昨收价")
+  private Double previousClose;
+  @Comment("涨跌额")
+  private Double priceChangeAmount;
+  @Comment("涨跌幅")
+  private Double priceFluctuationRange;
+  @Comment("成交量")
+  private Double volume;
+  @Comment("成交额")
+  private Double turnover;
+  @Comment("除权因子")
+  private Double factor;
+
+  @ManyToOne
+  @ToString.Exclude
+  private Stock stock;
+}

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/repository/DailyRepository.java

@@ -0,0 +1,14 @@
+package com.lanyuanxiaoyao.leopard.server.repository;
+
+import com.lanyuanxiaoyao.leopard.server.entity.Daily;
+import com.lanyuanxiaoyao.service.template.repository.SimpleRepository;
+import java.time.LocalDate;
+import java.util.List;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface DailyRepository extends SimpleRepository<Daily> {
+  @Query("select distinct daily.tradeDate from Daily daily")
+  List<LocalDate> findDistinctTradeDate();
+}

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/DailyService.java

@@ -0,0 +1,22 @@
+package com.lanyuanxiaoyao.leopard.server.service;
+
+import com.lanyuanxiaoyao.leopard.server.entity.Daily;
+import com.lanyuanxiaoyao.leopard.server.repository.DailyRepository;
+import com.lanyuanxiaoyao.service.template.service.SimpleServiceSupport;
+import java.time.LocalDate;
+import java.util.List;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DailyService extends SimpleServiceSupport<Daily> {
+  private final DailyRepository dailyRepository;
+
+  public DailyService(DailyRepository repository) {
+    super(repository);
+    this.dailyRepository = repository;
+  }
+
+  public List<LocalDate> findDistinctTradeDate() {
+    return dailyRepository.findDistinctTradeDate();
+  }
+}

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/TuShareService.java

@@ -3,6 +3,8 @@ package com.lanyuanxiaoyao.leopard.server.service;
 import cn.hutool.http.HttpUtil;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
 import java.util.List;
 import java.util.Map;
 import lombok.SneakyThrows;
@@ -21,6 +23,7 @@ import org.springframework.stereotype.Service;
 public class TuShareService {
   private static final String API_URL = "https://api.tushare.pro";
   private static final String API_TOKEN = "64ebff4fa679167600b905ee45dd88e76f3963c0ff39157f3f085f0e";
+  public static final DateTimeFormatter TRADE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd");
 
   private final ObjectMapper mapper;
 
@@ -31,19 +34,61 @@ public class TuShareService {
   @SneakyThrows
   private String buildRequest(String apiName, Map<String, Object> params, List<String> fields) {
     return mapper.writeValueAsString(Map.of(
-        "api_name", apiName,
+      "api_name", apiName,
-        "token", API_TOKEN,
+      "token", API_TOKEN,
-        "params", params,
+      "params", params,
-        "fields", fields
+      "fields", fields
     ));
   }
 
   @SneakyThrows
   public TuShareResponse stockList() {
     var response = HttpUtil.post(API_URL, buildRequest(
-        "stock_basic",
+      "stock_basic",
-        Map.of("list_status", "D,P,L"),
+      Map.of("list_status", "L"),
-        List.of("ts_code", "name", "fullname", "exchange", "industry", "list_status")
+      List.of("ts_code", "name", "fullname", "exchange", "industry")
+    ));
+    var tuShareResponse = mapper.readValue(response, TuShareResponse.class);
+    if (tuShareResponse.code != 0) {
+      throw new RuntimeException(tuShareResponse.message);
+    }
+    return tuShareResponse;
+  }
+
+  @SneakyThrows
+  public TuShareResponse dailyList(LocalDate tradeDate) {
+    var response = HttpUtil.post(API_URL, buildRequest(
+      "daily",
+      Map.of("trade_date", tradeDate.format(TRADE_FORMAT)),
+      List.of("ts_code", "trade_date", "open", "high", "low", "close", "pre_close", "change", "pct_chg", "vol", "amount")
+    ));
+    var tuShareResponse = mapper.readValue(response, TuShareResponse.class);
+    if (tuShareResponse.code != 0) {
+      throw new RuntimeException(tuShareResponse.message);
+    }
+    return tuShareResponse;
+  }
+
+  @SneakyThrows
+  public TuShareResponse tradeDateList(String exchange) {
+    var response = HttpUtil.post(API_URL, buildRequest(
+      "trade_cal",
+      Map.of("exchange", exchange, "is_open", 1),
+      List.of("cal_date")
+    ));
+    var tuShareResponse = mapper.readValue(response, TuShareResponse.class);
+    if (tuShareResponse.code != 0) {
+      throw new RuntimeException(tuShareResponse.message);
+    }
+    return tuShareResponse;
+  }
+
+  @SneakyThrows
+  public TuShareResponse factorList(LocalDate tradeDate) {
+    var response = HttpUtil.post(API_URL, buildRequest(
+      "adj_factor",
+      Map.of("trade_date", tradeDate.format(TRADE_FORMAT)),
+      List.of("ts_code", "trade_date", "adj_factor")
     ));
     var tuShareResponse = mapper.readValue(response, TuShareResponse.class);
     if (tuShareResponse.code != 0) {
@@ -53,14 +98,14 @@ public class TuShareService {
   }
 
   public record TuShareResponse(
-      Integer code,
+    Integer code,
-      @JsonProperty("msg")
+    @JsonProperty("msg")
-      String message,
+    String message,
-      Data data
+    Data data
   ) {
     public record Data(
-        List<String> fields,
+      List<String> fields,
-        List<List<String>> items
+      List<List<String>> items
     ) {
     }
   }

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/TaskMonitorNodes.java

@@ -26,8 +26,8 @@ public class TaskMonitorNodes {
 
   @LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "task_start", nodeName = "任务开始", nodeType = NodeTypeEnum.COMMON)
   public void taskStart(NodeComponent node) {
-    var context = node.getContextBean(TaskMonitorContext.class);
+    try {
-    if (ObjectUtil.isNotNull(context)) {
+      var context = node.getContextBean(TaskMonitorContext.class);
       var task = new Task();
       task.setName(context.getTemplate().getName());
       task.setDescription(context.getTemplate().getDescription());
@@ -35,6 +35,8 @@ public class TaskMonitorNodes {
       task.setLaunchedTime(LocalDateTime.now());
       var taskId = taskService.save(task);
       context.setTaskId(taskId);
+    } catch (Exception exception) {
+      log.warn("Not in task", exception);
     }
   }
 

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateDailyNode.java

@@ -0,0 +1,101 @@
+package com.lanyuanxiaoyao.leopard.server.service.task;
+
+import cn.hutool.core.thread.ThreadUtil;
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.leopard.server.entity.Daily;
+import com.lanyuanxiaoyao.leopard.server.entity.Stock;
+import com.lanyuanxiaoyao.leopard.server.service.DailyService;
+import com.lanyuanxiaoyao.leopard.server.service.StockService;
+import com.lanyuanxiaoyao.leopard.server.service.TuShareService;
+import com.yomahub.liteflow.annotation.LiteflowComponent;
+import com.yomahub.liteflow.core.NodeComponent;
+import java.time.LocalDate;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.transaction.support.TransactionTemplate;
+
+@Slf4j
+@LiteflowComponent("update_daily")
+public class UpdateDailyNode extends NodeComponent {
+  private final StockService stockService;
+  private final DailyService dailyService;
+  private final TuShareService tuShareService;
+  private final TransactionTemplate transactionTemplate;
+
+  public UpdateDailyNode(StockService stockService, DailyService dailyService, TuShareService tuShareService, TransactionTemplate transactionTemplate) {
+    this.stockService = stockService;
+    this.dailyService = dailyService;
+    this.tuShareService = tuShareService;
+    this.transactionTemplate = transactionTemplate;
+  }
+
+  // @Transactional(rollbackOn = Throwable.class)
+  @Override
+  public void process() {
+    var tradeDates = new HashSet<LocalDate>();
+    for (String exchange : List.of("SSE", "SZSE", "BSE")) {
+      var response = tuShareService.tradeDateList(exchange);
+      for (List<String> item : response.data().items()) {
+        if (ObjectUtil.isNotEmpty(item) && StrUtil.isNotBlank(item.get(0))) {
+          tradeDates.add(LocalDate.parse(item.get(0), TuShareService.TRADE_FORMAT));
+        }
+      }
+    }
+    var existsTradeDates = dailyService.findDistinctTradeDate();
+    var nowDate = LocalDate.now();
+    var targetDates = tradeDates.stream()
+      .filter(date -> date.isBefore(nowDate) || date.isEqual(nowDate))
+      .filter(date -> !existsTradeDates.contains(date))
+      .sorted()
+      .toList();
+    log.info("Target: {}", targetDates);
+    var stocks = stockService.list();
+    var stocksMap = stocks.stream().collect(Collectors.toMap(Stock::getCode, stock -> stock));
+    for (LocalDate tradeDate : targetDates) {
+      log.info("Trade date: {}", tradeDate);
+      var factorResponse = tuShareService.factorList(tradeDate);
+      var factorMap = new HashMap<String, Double>();
+      for (List<String> item : factorResponse.data().items()) {
+        factorMap.put(item.get(0), Double.valueOf(item.get(2)));
+      }
+      log.info("Factor: {}", factorMap);
+
+      var response = tuShareService.dailyList(tradeDate);
+      transactionTemplate.execute(status -> {
+        try {
+          for (List<String> item : response.data().items()) {
+            var code = item.get(0);
+            if (stocksMap.containsKey(code)) {
+              var stock = stocksMap.get(code);
+              var factor = factorMap.get(code);
+              var daily = new Daily();
+              daily.setTradeDate(LocalDate.parse(item.get(1), TuShareService.TRADE_FORMAT));
+              daily.setOpen(Double.valueOf(item.get(2)));
+              daily.setHigh(Double.valueOf(item.get(3)));
+              daily.setLow(Double.valueOf(item.get(4)));
+              daily.setClose(Double.valueOf(item.get(5)));
+              daily.setPreviousClose(Double.valueOf(item.get(6)));
+              daily.setPriceChangeAmount(Double.valueOf(item.get(7)));
+              daily.setPriceFluctuationRange(Double.valueOf(item.get(8)));
+              daily.setVolume(Double.valueOf(item.get(9)));
+              daily.setTurnover(Double.valueOf(item.get(10)));
+              daily.setFactor(factor);
+              daily.setStock(stock);
+              log.info("Daily: {}", daily);
+              dailyService.save(daily);
+            }
+          }
+          return true;
+        } catch (Exception exception) {
+          status.setRollbackOnly();
+          return false;
+        }
+      });
+      ThreadUtil.safeSleep(1000);
+    }
+  }
+}

leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateStockNode.java

@@ -1,30 +1,31 @@
 package com.lanyuanxiaoyao.leopard.server.service.task;
 
 import cn.hutool.core.util.EnumUtil;
-import cn.hutool.core.util.StrUtil;
 import com.lanyuanxiaoyao.leopard.server.entity.Stock;
 import com.lanyuanxiaoyao.leopard.server.service.StockService;
 import com.lanyuanxiaoyao.leopard.server.service.TuShareService;
 import com.yomahub.liteflow.annotation.LiteflowComponent;
 import com.yomahub.liteflow.core.NodeComponent;
 import jakarta.transaction.Transactional;
+import java.util.HashSet;
 import java.util.stream.Collectors;
 
-@LiteflowComponent("update_stock_information")
+@LiteflowComponent("update_stock")
-public class UpdateStockInformationNode extends NodeComponent {
+public class UpdateStockNode extends NodeComponent {
   private final StockService stockService;
   private final TuShareService tuShareService;
 
-  public UpdateStockInformationNode(StockService stockService, TuShareService tuShareService) {
+  public UpdateStockNode(StockService stockService, TuShareService tuShareService) {
     this.stockService = stockService;
     this.tuShareService = tuShareService;
   }
 
   @Transactional(rollbackOn = Throwable.class)
   @Override
-  public void process() throws Exception {
+  public void process() {
     var stocks = stockService.list();
     var stocksMap = stocks.stream().collect(Collectors.toMap(Stock::getCode, stock -> stock));
+    var targetCodes = new HashSet<String>();
     tuShareService.stockList()
       .data()
       .items()
@@ -34,14 +35,12 @@ public class UpdateStockInformationNode extends NodeComponent {
         var fullname = item.get(2);
         var market = EnumUtil.fromString(Stock.Market.class, item.get(3));
         var industry = item.get(4);
-        var listed = StrUtil.equals("L", item.get(5));
         if (stocksMap.containsKey(code)) {
           var stock = stocksMap.get(code);
           stock.setName(name);
           stock.setFullname(fullname);
           stock.setMarket(market);
           stock.setIndustry(industry);
-          stock.setListed(listed);
         } else {
           var stock = new Stock();
           stock.setCode(code);
@@ -49,10 +48,16 @@ public class UpdateStockInformationNode extends NodeComponent {
           stock.setFullname(fullname);
           stock.setMarket(market);
           stock.setIndustry(industry);
-          stock.setListed(listed);
           stocks.add(stock);
         }
+        targetCodes.add(code);
       });
+    var deleteStocks = stocks.stream()
+      .filter(stock -> !targetCodes.contains(stock.getCode()))
+      .map(Stock::getId)
+      .toList();
+    stockService.remove(deleteStocks);
+
     stockService.save(stocks);
   }
 }

leopard-server/src/main/resources/flow.xml

@@ -2,6 +2,9 @@
 <!DOCTYPE flow PUBLIC  "liteflow" "https://liteflow.cc/liteflow.dtd">
 <flow>
   <chain id="update_stock_information">
-    CATCH(THEN(task_start, update_stock_information, task_end)).DO(task_error)
+    CATCH(THEN(task_start, update_stock, task_end)).DO(task_error)
+  </chain>
+  <chain id="update_daily_information">
+    CATCH(THEN(task_start, update_daily, task_end)).DO(task_error)
   </chain>
 </flow>

leopard-server/src/test/resources/tushare.http

@@ -45,3 +45,64 @@ Content-Type: application/json
     "total_revenue_to_parent"
   ]
 }
+
+### Get daily list
+POST {{api_url}}
+Content-Type: application/json
+
+{
+  "api_name": "daily",
+  "token": "{{api_key}}",
+  "params": {
+    "trade_date": "19901219"
+  },
+  "fields": [
+    "ts_code",
+    "trade_date",
+    "open",
+    "high",
+    "low",
+    "close",
+    "pre_close",
+    "change",
+    "pct_chg",
+    "vol",
+    "amount"
+  ]
+}
+
+### Get trade date
+POST {{api_url}}
+Content-Type: application/json
+
+{
+  "api_name": "trade_cal",
+  "token": "{{api_key}}",
+  "params": {
+    "exchange": "BSE",
+    "is_open": 1
+  },
+  "fields": [
+    "exchange",
+    "cal_date",
+    "is_open",
+    "pretrade_date"
+  ]
+}
+
+### Get Factor
+POST {{api_url}}
+Content-Type: application/json
+
+{
+  "api_name": "adj_factor",
+  "token": "{{api_key}}",
+  "params": {
+    "trade_date": "20241231"
+  },
+  "fields": [
+    "ts_code",
+    "trade_date",
+    "adj_factor"
+  ]
+}