diff --git a/.idea/data_source_mapping.xml b/.idea/data_source_mapping.xml new file mode 100644 index 0000000..cd6d0d2 --- /dev/null +++ b/.idea/data_source_mapping.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/LeopardServerApplication.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/LeopardServerApplication.java index 76794da..f493c2f 100644 --- a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/LeopardServerApplication.java +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/LeopardServerApplication.java @@ -1,6 +1,9 @@ package com.lanyuanxiaoyao.leopard.server; import com.blinkfox.fenix.EnableFenix; +import com.lanyuanxiaoyao.leopard.server.service.TuShareService; +import com.yomahub.liteflow.core.FlowExecutor; +import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; @@ -21,7 +24,14 @@ public class LeopardServerApplication implements ApplicationRunner { SpringApplication.run(LeopardServerApplication.class, args); } + @Resource + private FlowExecutor executor; + @Resource + private TuShareService tuShareService; + @Override public void run(ApplicationArguments args) { + executor.execute2RespWithEL("THEN(update_daily)"); + // executor.execute2RespWithEL("THEN(update_stock)"); } } diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/entity/Daily.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/entity/Daily.java new file mode 100644 index 0000000..07c7ed3 --- /dev/null +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/entity/Daily.java @@ -0,0 +1,58 @@ +package com.lanyuanxiaoyao.leopard.server.entity; + +import com.lanyuanxiaoyao.leopard.server.Constants; +import com.lanyuanxiaoyao.service.template.entity.SimpleEntity; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EntityListeners; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import java.time.LocalDate; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.FieldNameConstants; +import org.hibernate.annotations.Comment; +import org.hibernate.annotations.DynamicInsert; +import org.hibernate.annotations.DynamicUpdate; +import org.hibernate.annotations.SoftDelete; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +@Setter +@Getter +@ToString(callSuper = true) +@FieldNameConstants +@Entity +@SoftDelete +@DynamicUpdate +@DynamicInsert +@EntityListeners(AuditingEntityListener.class) +@Table(name = Constants.DATABASE_PREFIX + "daily") +public class Daily extends SimpleEntity { + @Column(nullable = false) + private LocalDate tradeDate; + @Comment("开盘价") + private Double open; + @Comment("最高价") + private Double high; + @Comment("最低价") + private Double low; + @Comment("收盘价") + private Double close; + @Comment("昨收价") + private Double previousClose; + @Comment("涨跌额") + private Double priceChangeAmount; + @Comment("涨跌幅") + private Double priceFluctuationRange; + @Comment("成交量") + private Double volume; + @Comment("成交额") + private Double turnover; + @Comment("除权因子") + private Double factor; + + @ManyToOne + @ToString.Exclude + private Stock stock; +} diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/repository/DailyRepository.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/repository/DailyRepository.java new file mode 100644 index 0000000..75ce150 --- /dev/null +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/repository/DailyRepository.java @@ -0,0 +1,14 @@ +package com.lanyuanxiaoyao.leopard.server.repository; + +import com.lanyuanxiaoyao.leopard.server.entity.Daily; +import com.lanyuanxiaoyao.service.template.repository.SimpleRepository; +import java.time.LocalDate; +import java.util.List; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +@Repository +public interface DailyRepository extends SimpleRepository { + @Query("select distinct daily.tradeDate from Daily daily") + List findDistinctTradeDate(); +} diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/DailyService.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/DailyService.java new file mode 100644 index 0000000..161824d --- /dev/null +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/DailyService.java @@ -0,0 +1,22 @@ +package com.lanyuanxiaoyao.leopard.server.service; + +import com.lanyuanxiaoyao.leopard.server.entity.Daily; +import com.lanyuanxiaoyao.leopard.server.repository.DailyRepository; +import com.lanyuanxiaoyao.service.template.service.SimpleServiceSupport; +import java.time.LocalDate; +import java.util.List; +import org.springframework.stereotype.Service; + +@Service +public class DailyService extends SimpleServiceSupport { + private final DailyRepository dailyRepository; + + public DailyService(DailyRepository repository) { + super(repository); + this.dailyRepository = repository; + } + + public List findDistinctTradeDate() { + return dailyRepository.findDistinctTradeDate(); + } +} diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/TuShareService.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/TuShareService.java index 424b019..0e69afa 100644 --- a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/TuShareService.java +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/TuShareService.java @@ -3,6 +3,8 @@ package com.lanyuanxiaoyao.leopard.server.service; import cn.hutool.http.HttpUtil; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; import lombok.SneakyThrows; @@ -21,6 +23,7 @@ import org.springframework.stereotype.Service; public class TuShareService { private static final String API_URL = "https://api.tushare.pro"; private static final String API_TOKEN = "64ebff4fa679167600b905ee45dd88e76f3963c0ff39157f3f085f0e"; + public static final DateTimeFormatter TRADE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd"); private final ObjectMapper mapper; @@ -31,19 +34,61 @@ public class TuShareService { @SneakyThrows private String buildRequest(String apiName, Map params, List fields) { return mapper.writeValueAsString(Map.of( - "api_name", apiName, - "token", API_TOKEN, - "params", params, - "fields", fields + "api_name", apiName, + "token", API_TOKEN, + "params", params, + "fields", fields )); } @SneakyThrows public TuShareResponse stockList() { var response = HttpUtil.post(API_URL, buildRequest( - "stock_basic", - Map.of("list_status", "D,P,L"), - List.of("ts_code", "name", "fullname", "exchange", "industry", "list_status") + "stock_basic", + Map.of("list_status", "L"), + List.of("ts_code", "name", "fullname", "exchange", "industry") + )); + var tuShareResponse = mapper.readValue(response, TuShareResponse.class); + if (tuShareResponse.code != 0) { + throw new RuntimeException(tuShareResponse.message); + } + return tuShareResponse; + } + + @SneakyThrows + public TuShareResponse dailyList(LocalDate tradeDate) { + var response = HttpUtil.post(API_URL, buildRequest( + "daily", + Map.of("trade_date", tradeDate.format(TRADE_FORMAT)), + List.of("ts_code", "trade_date", "open", "high", "low", "close", "pre_close", "change", "pct_chg", "vol", "amount") + )); + var tuShareResponse = mapper.readValue(response, TuShareResponse.class); + if (tuShareResponse.code != 0) { + throw new RuntimeException(tuShareResponse.message); + } + return tuShareResponse; + } + + @SneakyThrows + public TuShareResponse tradeDateList(String exchange) { + var response = HttpUtil.post(API_URL, buildRequest( + "trade_cal", + Map.of("exchange", exchange, "is_open", 1), + List.of("cal_date") + )); + var tuShareResponse = mapper.readValue(response, TuShareResponse.class); + if (tuShareResponse.code != 0) { + throw new RuntimeException(tuShareResponse.message); + } + return tuShareResponse; + } + + @SneakyThrows + public TuShareResponse factorList(LocalDate tradeDate) { + var response = HttpUtil.post(API_URL, buildRequest( + "adj_factor", + Map.of("trade_date", tradeDate.format(TRADE_FORMAT)), + List.of("ts_code", "trade_date", "adj_factor") )); var tuShareResponse = mapper.readValue(response, TuShareResponse.class); if (tuShareResponse.code != 0) { @@ -53,14 +98,14 @@ public class TuShareService { } public record TuShareResponse( - Integer code, - @JsonProperty("msg") - String message, - Data data + Integer code, + @JsonProperty("msg") + String message, + Data data ) { public record Data( - List fields, - List> items + List fields, + List> items ) { } } diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/TaskMonitorNodes.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/TaskMonitorNodes.java index 9c6f77d..809c266 100644 --- a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/TaskMonitorNodes.java +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/TaskMonitorNodes.java @@ -26,8 +26,8 @@ public class TaskMonitorNodes { @LiteflowMethod(value = LiteFlowMethodEnum.PROCESS, nodeId = "task_start", nodeName = "任务开始", nodeType = NodeTypeEnum.COMMON) public void taskStart(NodeComponent node) { - var context = node.getContextBean(TaskMonitorContext.class); - if (ObjectUtil.isNotNull(context)) { + try { + var context = node.getContextBean(TaskMonitorContext.class); var task = new Task(); task.setName(context.getTemplate().getName()); task.setDescription(context.getTemplate().getDescription()); @@ -35,6 +35,8 @@ public class TaskMonitorNodes { task.setLaunchedTime(LocalDateTime.now()); var taskId = taskService.save(task); context.setTaskId(taskId); + } catch (Exception exception) { + log.warn("Not in task", exception); } } diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateDailyNode.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateDailyNode.java new file mode 100644 index 0000000..d90b9f1 --- /dev/null +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateDailyNode.java @@ -0,0 +1,101 @@ +package com.lanyuanxiaoyao.leopard.server.service.task; + +import cn.hutool.core.thread.ThreadUtil; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; +import com.lanyuanxiaoyao.leopard.server.entity.Daily; +import com.lanyuanxiaoyao.leopard.server.entity.Stock; +import com.lanyuanxiaoyao.leopard.server.service.DailyService; +import com.lanyuanxiaoyao.leopard.server.service.StockService; +import com.lanyuanxiaoyao.leopard.server.service.TuShareService; +import com.yomahub.liteflow.annotation.LiteflowComponent; +import com.yomahub.liteflow.core.NodeComponent; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.transaction.support.TransactionTemplate; + +@Slf4j +@LiteflowComponent("update_daily") +public class UpdateDailyNode extends NodeComponent { + private final StockService stockService; + private final DailyService dailyService; + private final TuShareService tuShareService; + private final TransactionTemplate transactionTemplate; + + public UpdateDailyNode(StockService stockService, DailyService dailyService, TuShareService tuShareService, TransactionTemplate transactionTemplate) { + this.stockService = stockService; + this.dailyService = dailyService; + this.tuShareService = tuShareService; + this.transactionTemplate = transactionTemplate; + } + + // @Transactional(rollbackOn = Throwable.class) + @Override + public void process() { + var tradeDates = new HashSet(); + for (String exchange : List.of("SSE", "SZSE", "BSE")) { + var response = tuShareService.tradeDateList(exchange); + for (List item : response.data().items()) { + if (ObjectUtil.isNotEmpty(item) && StrUtil.isNotBlank(item.get(0))) { + tradeDates.add(LocalDate.parse(item.get(0), TuShareService.TRADE_FORMAT)); + } + } + } + var existsTradeDates = dailyService.findDistinctTradeDate(); + var nowDate = LocalDate.now(); + var targetDates = tradeDates.stream() + .filter(date -> date.isBefore(nowDate) || date.isEqual(nowDate)) + .filter(date -> !existsTradeDates.contains(date)) + .sorted() + .toList(); + log.info("Target: {}", targetDates); + var stocks = stockService.list(); + var stocksMap = stocks.stream().collect(Collectors.toMap(Stock::getCode, stock -> stock)); + for (LocalDate tradeDate : targetDates) { + log.info("Trade date: {}", tradeDate); + var factorResponse = tuShareService.factorList(tradeDate); + var factorMap = new HashMap(); + for (List item : factorResponse.data().items()) { + factorMap.put(item.get(0), Double.valueOf(item.get(2))); + } + log.info("Factor: {}", factorMap); + + var response = tuShareService.dailyList(tradeDate); + transactionTemplate.execute(status -> { + try { + for (List item : response.data().items()) { + var code = item.get(0); + if (stocksMap.containsKey(code)) { + var stock = stocksMap.get(code); + var factor = factorMap.get(code); + var daily = new Daily(); + daily.setTradeDate(LocalDate.parse(item.get(1), TuShareService.TRADE_FORMAT)); + daily.setOpen(Double.valueOf(item.get(2))); + daily.setHigh(Double.valueOf(item.get(3))); + daily.setLow(Double.valueOf(item.get(4))); + daily.setClose(Double.valueOf(item.get(5))); + daily.setPreviousClose(Double.valueOf(item.get(6))); + daily.setPriceChangeAmount(Double.valueOf(item.get(7))); + daily.setPriceFluctuationRange(Double.valueOf(item.get(8))); + daily.setVolume(Double.valueOf(item.get(9))); + daily.setTurnover(Double.valueOf(item.get(10))); + daily.setFactor(factor); + daily.setStock(stock); + log.info("Daily: {}", daily); + dailyService.save(daily); + } + } + return true; + } catch (Exception exception) { + status.setRollbackOnly(); + return false; + } + }); + ThreadUtil.safeSleep(1000); + } + } +} diff --git a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateStockInformationNode.java b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateStockNode.java similarity index 77% rename from leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateStockInformationNode.java rename to leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateStockNode.java index 1741046..33aacdf 100644 --- a/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateStockInformationNode.java +++ b/leopard-server/src/main/java/com/lanyuanxiaoyao/leopard/server/service/task/UpdateStockNode.java @@ -1,30 +1,31 @@ package com.lanyuanxiaoyao.leopard.server.service.task; import cn.hutool.core.util.EnumUtil; -import cn.hutool.core.util.StrUtil; import com.lanyuanxiaoyao.leopard.server.entity.Stock; import com.lanyuanxiaoyao.leopard.server.service.StockService; import com.lanyuanxiaoyao.leopard.server.service.TuShareService; import com.yomahub.liteflow.annotation.LiteflowComponent; import com.yomahub.liteflow.core.NodeComponent; import jakarta.transaction.Transactional; +import java.util.HashSet; import java.util.stream.Collectors; -@LiteflowComponent("update_stock_information") -public class UpdateStockInformationNode extends NodeComponent { +@LiteflowComponent("update_stock") +public class UpdateStockNode extends NodeComponent { private final StockService stockService; private final TuShareService tuShareService; - public UpdateStockInformationNode(StockService stockService, TuShareService tuShareService) { + public UpdateStockNode(StockService stockService, TuShareService tuShareService) { this.stockService = stockService; this.tuShareService = tuShareService; } @Transactional(rollbackOn = Throwable.class) @Override - public void process() throws Exception { + public void process() { var stocks = stockService.list(); var stocksMap = stocks.stream().collect(Collectors.toMap(Stock::getCode, stock -> stock)); + var targetCodes = new HashSet(); tuShareService.stockList() .data() .items() @@ -34,14 +35,12 @@ public class UpdateStockInformationNode extends NodeComponent { var fullname = item.get(2); var market = EnumUtil.fromString(Stock.Market.class, item.get(3)); var industry = item.get(4); - var listed = StrUtil.equals("L", item.get(5)); if (stocksMap.containsKey(code)) { var stock = stocksMap.get(code); stock.setName(name); stock.setFullname(fullname); stock.setMarket(market); stock.setIndustry(industry); - stock.setListed(listed); } else { var stock = new Stock(); stock.setCode(code); @@ -49,10 +48,16 @@ public class UpdateStockInformationNode extends NodeComponent { stock.setFullname(fullname); stock.setMarket(market); stock.setIndustry(industry); - stock.setListed(listed); stocks.add(stock); } + targetCodes.add(code); }); + var deleteStocks = stocks.stream() + .filter(stock -> !targetCodes.contains(stock.getCode())) + .map(Stock::getId) + .toList(); + stockService.remove(deleteStocks); + stockService.save(stocks); } } diff --git a/leopard-server/src/main/resources/flow.xml b/leopard-server/src/main/resources/flow.xml index a7d4480..a4a28c8 100644 --- a/leopard-server/src/main/resources/flow.xml +++ b/leopard-server/src/main/resources/flow.xml @@ -2,6 +2,9 @@ - CATCH(THEN(task_start, update_stock_information, task_end)).DO(task_error) + CATCH(THEN(task_start, update_stock, task_end)).DO(task_error) + + + CATCH(THEN(task_start, update_daily, task_end)).DO(task_error) diff --git a/leopard-server/src/test/resources/tushare.http b/leopard-server/src/test/resources/tushare.http index 8f8f8ca..ae4b472 100644 --- a/leopard-server/src/test/resources/tushare.http +++ b/leopard-server/src/test/resources/tushare.http @@ -45,3 +45,64 @@ Content-Type: application/json "total_revenue_to_parent" ] } + +### Get daily list +POST {{api_url}} +Content-Type: application/json + +{ + "api_name": "daily", + "token": "{{api_key}}", + "params": { + "trade_date": "19901219" + }, + "fields": [ + "ts_code", + "trade_date", + "open", + "high", + "low", + "close", + "pre_close", + "change", + "pct_chg", + "vol", + "amount" + ] +} + +### Get trade date +POST {{api_url}} +Content-Type: application/json + +{ + "api_name": "trade_cal", + "token": "{{api_key}}", + "params": { + "exchange": "BSE", + "is_open": 1 + }, + "fields": [ + "exchange", + "cal_date", + "is_open", + "pretrade_date" + ] +} + +### Get Factor +POST {{api_url}} +Content-Type: application/json + +{ + "api_name": "adj_factor", + "token": "{{api_key}}", + "params": { + "trade_date": "20241231" + }, + "fields": [ + "ts_code", + "trade_date", + "adj_factor" + ] +}