From f0a122d86215d4e5891a1ab5df31e91835486e02 Mon Sep 17 00:00:00 2001 From: lijiahang Date: Tue, 25 Jun 2024 10:42:32 +0800 Subject: [PATCH] =?UTF-8?q?:zap:=20=E8=87=AA=E5=8A=A8=E6=B8=85=E7=90=86?= =?UTF-8?q?=E9=85=8D=E7=BD=AE.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/constant/ErrorMessage.java | 6 ++ .../common/entity/AutoClearConfig.java | 25 +++++++ .../config/LocalStorageConfig.java | 4 + .../config/LogsStorageConfig.java | 4 + .../src/main/resources/application.yaml | 14 +++- .../config/AppExecLogAutoClearConfig.java | 22 ++++++ .../asset/define/config/AppExecLogConfig.java | 12 --- .../AppHostConnectLogAutoClearConfig.java | 22 ++++++ .../request/exec/ExecLogQueryRequest.java | 8 ++ .../host/HostConnectLogQueryRequest.java | 7 ++ .../module/asset/enums/ExecStatusEnum.java | 5 ++ .../asset/enums/HostConnectStatusEnum.java | 6 ++ .../handler/TerminalCheckHandler.java | 21 +++--- .../module/asset/service/ExecLogService.java | 15 ++++ .../asset/service/HostConnectLogService.java | 5 +- .../service/impl/ExecLogServiceImpl.java | 73 +++++++++++-------- .../impl/HostConnectLogServiceImpl.java | 35 ++++++--- .../asset/task/ExecLogFileAutoClearTask.java | 70 +++++------------- .../task/HostConnectLogAutoClearTask.java | 66 +++++++++++++++++ ...itional-spring-configuration-metadata.json | 19 ++++- 20 files changed, 318 insertions(+), 121 deletions(-) create mode 100644 orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/entity/AutoClearConfig.java create mode 100644 orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogAutoClearConfig.java create mode 100644 orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppHostConnectLogAutoClearConfig.java create mode 100644 orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/HostConnectLogAutoClearTask.java diff --git a/orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/constant/ErrorMessage.java b/orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/constant/ErrorMessage.java index c3baf40f..1d638d57 100644 --- a/orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/constant/ErrorMessage.java +++ b/orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/constant/ErrorMessage.java @@ -83,8 +83,12 @@ public interface ErrorMessage { String UNKNOWN_TYPE = "未知类型"; + String ERROR_TYPE = "错误的类型"; + String FILE_ABSENT = "文件不存在"; + String FILE_ABSENT_CLEAR = "文件不存在 (可能已被清理)"; + String LOG_ABSENT = "日志不存在"; String TASK_ABSENT = "任务不存在"; @@ -95,6 +99,8 @@ public interface ErrorMessage { String FILE_READ_ERROR = "文件读取失败"; + String FILE_READ_ERROR_CLEAR = "文件读取失败 (可能已被清理)"; + String PLEASE_CHECK_HOST_SSH = "请检查主机 {} 是否存在/权限/SSH配置"; String CLIENT_ABORT = "手动中断"; diff --git a/orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/entity/AutoClearConfig.java b/orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/entity/AutoClearConfig.java new file mode 100644 index 00000000..d98f2e70 --- /dev/null +++ b/orion-visor-framework/orion-visor-framework-common/src/main/java/com/orion/visor/framework/common/entity/AutoClearConfig.java @@ -0,0 +1,25 @@ +package com.orion.visor.framework.common.entity; + +import lombok.Data; + +/** + * 自动清理配置 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2024/6/24 15:03 + */ +@Data +public class AutoClearConfig { + + /** + * 是否开启 + */ + private Boolean enabled; + + /** + * 保留周期 (天) + */ + private Integer keepPeriod; + +} diff --git a/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LocalStorageConfig.java b/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LocalStorageConfig.java index 8b028ebe..80934dc2 100644 --- a/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LocalStorageConfig.java +++ b/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LocalStorageConfig.java @@ -1,6 +1,8 @@ package com.orion.visor.framework.storage.configuration.config; import com.orion.visor.framework.storage.core.client.local.LocalFileClientConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -10,6 +12,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties; * @version 1.0.0 * @since 2023/6/30 18:40 */ +@Data +@EqualsAndHashCode(callSuper = true) @ConfigurationProperties(prefix = "orion.storage.local") public class LocalStorageConfig extends LocalFileClientConfig { diff --git a/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LogsStorageConfig.java b/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LogsStorageConfig.java index b6214417..337c2ea4 100644 --- a/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LogsStorageConfig.java +++ b/orion-visor-framework/orion-visor-spring-boot-starter-storage/src/main/java/com/orion/visor/framework/storage/configuration/config/LogsStorageConfig.java @@ -1,6 +1,8 @@ package com.orion.visor.framework.storage.configuration.config; import com.orion.visor.framework.storage.core.client.local.LocalFileClientConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.springframework.boot.context.properties.ConfigurationProperties; /** @@ -10,6 +12,8 @@ import org.springframework.boot.context.properties.ConfigurationProperties; * @version 1.0.0 * @since 2023/6/30 18:40 */ +@Data +@EqualsAndHashCode(callSuper = true) @ConfigurationProperties(prefix = "orion.storage.logs") public class LogsStorageConfig extends LocalFileClientConfig { diff --git a/orion-visor-launch/src/main/resources/application.yaml b/orion-visor-launch/src/main/resources/application.yaml index 855dfb64..3edfe856 100644 --- a/orion-visor-launch/src/main/resources/application.yaml +++ b/orion-visor-launch/src/main/resources/application.yaml @@ -189,10 +189,16 @@ app: exec-log: # 是否拼接 ansi 执行状态日志 append-ansi: true - # 自动清理执行文件 - auto-clear: true - # 保留周期 (天) - keep-period: 30 + # 自动清理配置 + auto-clear: + # 批量执行日志 + exec-log: + enabled: false + keep-period: 30 + # 主机连接日志 + host-connect-log: + enabled: false + keep-period: 30 # orion framework config orion: diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogAutoClearConfig.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogAutoClearConfig.java new file mode 100644 index 00000000..da4a5d68 --- /dev/null +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogAutoClearConfig.java @@ -0,0 +1,22 @@ +package com.orion.visor.module.asset.define.config; + +import com.orion.visor.framework.common.entity.AutoClearConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * 批量执行日志自动清理配置 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2024/6/24 15:01 + */ +@Data +@Component +@EqualsAndHashCode(callSuper = true) +@ConfigurationProperties(prefix = "app.auto-clear.exec-log") +public class AppExecLogAutoClearConfig extends AutoClearConfig { + +} diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogConfig.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogConfig.java index aa6f4c0e..d2262949 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogConfig.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppExecLogConfig.java @@ -21,20 +21,8 @@ public class AppExecLogConfig { */ private Boolean appendAnsi; - /** - * 自动清理执行文件 - */ - private Boolean autoClear; - - /** - * 保留周期 (天) - */ - private Integer keepPeriod; - public AppExecLogConfig() { this.appendAnsi = true; - this.autoClear = true; - this.keepPeriod = 30; } } diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppHostConnectLogAutoClearConfig.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppHostConnectLogAutoClearConfig.java new file mode 100644 index 00000000..3c7a2906 --- /dev/null +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/define/config/AppHostConnectLogAutoClearConfig.java @@ -0,0 +1,22 @@ +package com.orion.visor.module.asset.define.config; + +import com.orion.visor.framework.common.entity.AutoClearConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * 主机连接日志自动清理配置 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2024/6/24 15:01 + */ +@Data +@Component +@EqualsAndHashCode(callSuper = true) +@ConfigurationProperties(prefix = "app.auto-clear.host-connect-log") +public class AppHostConnectLogAutoClearConfig extends AutoClearConfig { + +} diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/exec/ExecLogQueryRequest.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/exec/ExecLogQueryRequest.java index f872735e..4e42d0d3 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/exec/ExecLogQueryRequest.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/exec/ExecLogQueryRequest.java @@ -7,6 +7,7 @@ import lombok.*; import javax.validation.constraints.Size; import java.util.Date; +import java.util.List; /** * 批量执行日志 查询请求对象 @@ -54,4 +55,11 @@ public class ExecLogQueryRequest extends PageRequest { @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date[] startTimeRange; + @Schema(description = "创建时间 <=") + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date createTimeLe; + + @Schema(description = "状态") + private List statusList; + } diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/host/HostConnectLogQueryRequest.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/host/HostConnectLogQueryRequest.java index df786c32..5d33a3b6 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/host/HostConnectLogQueryRequest.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/entity/request/host/HostConnectLogQueryRequest.java @@ -56,4 +56,11 @@ public class HostConnectLogQueryRequest extends PageRequest { @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date[] startTimeRange; + @Schema(description = "创建时间 <=") + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date createTimeLe; + + @Schema(description = "状态") + private List statusList; + } diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/ExecStatusEnum.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/ExecStatusEnum.java index cbbf5451..a8516db2 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/ExecStatusEnum.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/ExecStatusEnum.java @@ -1,8 +1,11 @@ package com.orion.visor.module.asset.enums; +import com.orion.lang.utils.collect.Lists; import lombok.AllArgsConstructor; import lombok.Getter; +import java.util.List; + /** * 批量执行状态 * @@ -36,6 +39,8 @@ public enum ExecStatusEnum { ; + public static final List FINISH_STATUS_LIST = Lists.of(COMPLETED.name(), FAILED.name()); + private final boolean closeable; public static ExecStatusEnum of(String status) { diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/HostConnectStatusEnum.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/HostConnectStatusEnum.java index c9a86f27..892eaffa 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/HostConnectStatusEnum.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/enums/HostConnectStatusEnum.java @@ -1,5 +1,9 @@ package com.orion.visor.module.asset.enums; +import com.orion.lang.utils.collect.Lists; + +import java.util.List; + /** * 主机连接状态 * @@ -31,6 +35,8 @@ public enum HostConnectStatusEnum { ; + public static final List FINISH_STATUS_LIST = Lists.of(COMPLETE.name(), FAILED.name(), FORCE_OFFLINE.name()); + public static HostConnectStatusEnum of(String type) { if (type == null) { return null; diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/handler/host/terminal/handler/TerminalCheckHandler.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/handler/host/terminal/handler/TerminalCheckHandler.java index a5583943..7e1f8426 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/handler/host/terminal/handler/TerminalCheckHandler.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/handler/host/terminal/handler/TerminalCheckHandler.java @@ -13,6 +13,7 @@ import com.orion.visor.framework.common.enums.BooleanBit; import com.orion.visor.framework.websocket.core.utils.WebSockets; import com.orion.visor.module.asset.dao.HostDAO; import com.orion.visor.module.asset.define.operator.HostTerminalOperatorType; +import com.orion.visor.module.asset.entity.domain.HostConnectLogDO; import com.orion.visor.module.asset.entity.domain.HostDO; import com.orion.visor.module.asset.entity.dto.HostTerminalConnectDTO; import com.orion.visor.module.asset.entity.request.host.HostConnectLogCreateRequest; @@ -95,9 +96,9 @@ public class TerminalCheckHandler extends AbstractTerminalHandler idList, String source); + /** + * 批量删除批量执行日志 + * + * @param idList idList + * @return effect + */ + Integer deleteExecLogByIdList(List idList); + /** * 查询批量执行日志数量 * @@ -128,4 +136,11 @@ public interface ExecLogService { */ void downloadLogFile(Long id, String source, HttpServletResponse response); + /** + * 异步删除日志文件 + * + * @param idList idList + */ + void asyncDeleteLogFiles(List idList); + } diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/HostConnectLogService.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/HostConnectLogService.java index 38bbb7b2..f138be3b 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/HostConnectLogService.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/HostConnectLogService.java @@ -1,6 +1,7 @@ package com.orion.visor.module.asset.service; import com.orion.lang.define.wrapper.DataGrid; +import com.orion.visor.module.asset.entity.domain.HostConnectLogDO; import com.orion.visor.module.asset.entity.request.host.HostConnectLogCreateRequest; import com.orion.visor.module.asset.entity.request.host.HostConnectLogQueryRequest; import com.orion.visor.module.asset.entity.vo.HostConnectLogVO; @@ -25,9 +26,9 @@ public interface HostConnectLogService { * * @param type type * @param request request - * @return id + * @return record */ - Long create(HostConnectTypeEnum type, HostConnectLogCreateRequest request); + HostConnectLogDO create(HostConnectTypeEnum type, HostConnectLogCreateRequest request); /** * 分页查询主机连接日志 diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/ExecLogServiceImpl.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/ExecLogServiceImpl.java index 9fd13b7e..36998bd0 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/ExecLogServiceImpl.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/ExecLogServiceImpl.java @@ -11,11 +11,13 @@ import com.orion.lang.utils.Strings; import com.orion.lang.utils.collect.Lists; import com.orion.lang.utils.io.Files1; import com.orion.lang.utils.io.Streams; +import com.orion.spring.SpringHolder; import com.orion.visor.framework.biz.operator.log.core.utils.OperatorLogs; import com.orion.visor.framework.common.annotation.Keep; import com.orion.visor.framework.common.constant.Const; import com.orion.visor.framework.common.constant.ErrorMessage; -import com.orion.visor.framework.common.constant.PathConst; +import com.orion.visor.framework.common.constant.FileConst; +import com.orion.visor.framework.common.enums.EndpointDefine; import com.orion.visor.framework.common.file.FileClient; import com.orion.visor.framework.common.utils.Valid; import com.orion.visor.framework.redis.core.utils.RedisStrings; @@ -46,11 +48,13 @@ import com.orion.visor.module.asset.service.ExecLogService; import com.orion.visor.module.asset.service.HostConfigService; import com.orion.web.servlet.web.Servlets; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; import javax.servlet.http.HttpServletResponse; +import java.io.File; import java.io.InputStream; import java.util.*; import java.util.stream.Collectors; @@ -173,20 +177,7 @@ public class ExecLogServiceImpl implements ExecLogService { @Override @Transactional(rollbackFor = Exception.class) public Integer deleteExecLogById(Long id, String source) { - log.info("ExecLogService-deleteExecLogById id: {}", id); - // 检查数据是否存在 - ExecLogDO record = execLogDAO.selectByIdSource(id, source); - Valid.notNull(record, ErrorMessage.DATA_ABSENT); - // 中断命令执行 - this.interruptTask(Lists.singleton(id)); - // 删除执行日志 - int effect = execLogDAO.deleteById(id); - // 删除主机日志 - execHostLogService.deleteExecHostLogByLogId(Lists.singleton(id)); - log.info("ExecLogService-deleteExecLogById id: {}, effect: {}", id, effect); - // 设置日志参数 - OperatorLogs.add(OperatorLogs.COUNT, effect); - return effect; + return this.deleteExecLogByIdList(Lists.singleton(id), source); } @Override @@ -201,15 +192,28 @@ public class ExecLogServiceImpl implements ExecLogService { .count() .intValue(); Valid.isTrue(idList.size() == count, ErrorMessage.DATA_MODIFIED); + // 删除 + return this.deleteExecLogByIdList(idList); + } + + @Override + public Integer deleteExecLogByIdList(List idList) { + log.info("ExecLogService-deleteExecLogByIdList start: {}", idList); + if (Lists.isEmpty(idList)) { + OperatorLogs.add(OperatorLogs.COUNT, Const.N_0); + return Const.N_0; + } // 中断命令执行 this.interruptTask(idList); // 删除执行日志 int effect = execLogDAO.deleteBatchIds(idList); // 删除主机日志 execHostLogService.deleteExecHostLogByLogId(idList); - log.info("ExecLogService-deleteExecLogByIdList effect: {}", effect); + log.info("ExecLogService-deleteExecLogByIdList end effect: {}", effect); // 设置日志参数 OperatorLogs.add(OperatorLogs.COUNT, effect); + // 异步删除文件 + SpringHolder.getBean(ExecLogService.class).asyncDeleteLogFiles(idList); return effect; } @@ -229,19 +233,8 @@ public class ExecLogServiceImpl implements ExecLogService { .stream() .map(ExecLogDO::getId) .collect(Collectors.toList()); - int effect = 0; - if (!idList.isEmpty()) { - // 中断命令执行 - this.interruptTask(idList); - // 删除执行日志 - effect = execLogDAO.delete(wrapper); - // 删除主机日志 - execHostLogService.deleteExecHostLogByLogId(idList); - } - log.info("ExecLogService.clearExecLog finish {}", effect); - // 设置日志参数 - OperatorLogs.add(OperatorLogs.COUNT, effect); - return effect; + // 删除 + return this.deleteExecLogByIdList(idList); } @Override @@ -428,19 +421,35 @@ public class ExecLogServiceImpl implements ExecLogService { } catch (Exception e) { log.error("ExecLogService.downloadLogFile error id: {}", id, e); Streams.close(in); - String errorMessage = ErrorMessage.FILE_READ_ERROR; + String errorMessage = ErrorMessage.FILE_READ_ERROR_CLEAR; if (e instanceof InvalidArgumentException) { errorMessage = e.getMessage(); } // 响应错误信息 try { - Servlets.transfer(response, Strings.bytes(errorMessage), PathConst.ERROR_LOG); + Servlets.transfer(response, Strings.bytes(errorMessage), FileConst.ERROR_LOG); } catch (Exception ex) { log.error("ExecLogService.downloadLogFile transfer-error id: {}", id, ex); } } } + @Override + @Async("asyncExecutor") + public void asyncDeleteLogFiles(List idList) { + if (Lists.isEmpty(idList)) { + return; + } + // 删除 + idList.stream() + .map(s -> EndpointDefine.EXEC_LOG.format(s, Const.EMPTY)) + .map(logsFileClient::getReturnPath) + .map(logsFileClient::getAbsolutePath) + .map(Files1::getParentPath) + .map(File::new) + .forEach(Files1::delete); + } + /** * 构建查询 wrapper * @@ -457,8 +466,10 @@ public class ExecLogServiceImpl implements ExecLogService { .like(ExecLogDO::getDescription, request.getDescription()) .like(ExecLogDO::getCommand, request.getCommand()) .eq(ExecLogDO::getStatus, request.getStatus()) + .in(ExecLogDO::getStatus, request.getStatusList()) .ge(ExecLogDO::getStartTime, Arrays1.getIfPresent(request.getStartTimeRange(), 0)) .le(ExecLogDO::getStartTime, Arrays1.getIfPresent(request.getStartTimeRange(), 1)) + .le(ExecLogDO::getCreateTime, request.getCreateTimeLe()) .orderByDesc(ExecLogDO::getId); } diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/HostConnectLogServiceImpl.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/HostConnectLogServiceImpl.java index 38dae986..bac1b741 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/HostConnectLogServiceImpl.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/service/impl/HostConnectLogServiceImpl.java @@ -2,13 +2,13 @@ package com.orion.visor.module.asset.service.impl; import com.alibaba.fastjson.JSON; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import com.orion.lang.constant.Const; import com.orion.lang.define.wrapper.DataGrid; import com.orion.lang.utils.Arrays1; -import com.orion.lang.utils.Valid; import com.orion.lang.utils.collect.Lists; import com.orion.visor.framework.biz.operator.log.core.utils.OperatorLogs; +import com.orion.visor.framework.common.constant.Const; import com.orion.visor.framework.common.constant.ErrorMessage; +import com.orion.visor.framework.common.utils.Valid; import com.orion.visor.framework.security.core.utils.SecurityUtils; import com.orion.visor.module.asset.convert.HostConnectLogConvert; import com.orion.visor.module.asset.dao.HostConnectLogDAO; @@ -52,7 +52,7 @@ public class HostConnectLogServiceImpl implements HostConnectLogService { private HostTerminalManager hostTerminalManager; @Override - public Long create(HostConnectTypeEnum type, HostConnectLogCreateRequest request) { + public HostConnectLogDO create(HostConnectTypeEnum type, HostConnectLogCreateRequest request) { HostConnectLogDO record = HostConnectLogConvert.MAPPER.to(request); record.setType(type.name()); String status = request.getStatus(); @@ -64,7 +64,7 @@ public class HostConnectLogServiceImpl implements HostConnectLogService { record.setEndTime(new Date()); } hostConnectLogDAO.insert(record); - return record.getId(); + return record; } @Override @@ -169,6 +169,11 @@ public class HostConnectLogServiceImpl implements HostConnectLogService { @Override public Integer deleteHostConnectLog(List idList) { log.info("HostConnectLogService.deleteHostConnectLog start {}", JSON.toJSONString(idList)); + if (Lists.isEmpty(idList)) { + OperatorLogs.add(OperatorLogs.COUNT, Const.N_0); + return Const.N_0; + } + // 删除 int effect = hostConnectLogDAO.deleteBatchIds(idList); log.info("HostConnectLogService.deleteHostConnectLog finish {}", effect); // 设置日志参数 @@ -184,13 +189,21 @@ public class HostConnectLogServiceImpl implements HostConnectLogService { @Override public Integer clearHostConnectLog(HostConnectLogQueryRequest request) { log.info("HostConnectLogService.clearHostConnectLog start {}", JSON.toJSONString(request)); + // 查询 + LambdaQueryWrapper wrapper = this.buildQueryWrapper(request) + .select(HostConnectLogDO::getId); + List list = hostConnectLogDAO.selectList(wrapper); + if (list.isEmpty()) { + log.info("HostConnectLogService.clearHostConnectLog empty"); + // 设置日志参数 + OperatorLogs.add(OperatorLogs.COUNT, Const.N_0); + return Const.N_0; + } // 删除 - LambdaQueryWrapper wrapper = this.buildQueryWrapper(request); - int effect = hostConnectLogDAO.delete(wrapper); - log.info("HostConnectLogService.clearHostConnectLog finish {}", effect); - // 设置日志参数 - OperatorLogs.add(OperatorLogs.COUNT, effect); - return effect; + List idList = list.stream() + .map(HostConnectLogDO::getId) + .collect(Collectors.toList()); + return this.deleteHostConnectLog(idList); } @Override @@ -229,8 +242,10 @@ public class HostConnectLogServiceImpl implements HostConnectLogService { .eq(HostConnectLogDO::getType, request.getType()) .like(HostConnectLogDO::getToken, request.getToken()) .eq(HostConnectLogDO::getStatus, request.getStatus()) + .in(HostConnectLogDO::getStatus, request.getStatusList()) .ge(HostConnectLogDO::getStartTime, Arrays1.getIfPresent(request.getStartTimeRange(), 0)) .le(HostConnectLogDO::getStartTime, Arrays1.getIfPresent(request.getStartTimeRange(), 1)) + .le(HostConnectLogDO::getCreateTime, request.getCreateTimeLe()) .orderByDesc(HostConnectLogDO::getId); } diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/ExecLogFileAutoClearTask.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/ExecLogFileAutoClearTask.java index b5aa18cb..ebe1cccb 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/ExecLogFileAutoClearTask.java +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/ExecLogFileAutoClearTask.java @@ -1,22 +1,18 @@ package com.orion.visor.module.asset.task; -import com.orion.lang.utils.Strings; -import com.orion.lang.utils.io.Files1; import com.orion.lang.utils.time.Dates; -import com.orion.visor.framework.common.annotation.Keep; -import com.orion.visor.framework.common.file.FileClient; import com.orion.visor.framework.common.utils.LockerUtils; -import com.orion.visor.module.asset.dao.ExecHostLogDAO; -import com.orion.visor.module.asset.define.config.AppExecLogConfig; -import com.orion.visor.module.asset.entity.domain.ExecHostLogDO; +import com.orion.visor.module.asset.define.config.AppExecLogAutoClearConfig; +import com.orion.visor.module.asset.entity.request.exec.ExecLogQueryRequest; +import com.orion.visor.module.asset.enums.ExecStatusEnum; +import com.orion.visor.module.asset.service.ExecLogService; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.Resource; -import java.io.File; -import java.util.List; +import java.util.Date; /** * 执行日志文件自动清理 @@ -27,23 +23,19 @@ import java.util.List; */ @Slf4j @Component -@ConditionalOnProperty(value = "app.exec-log.auto-clear", havingValue = "true", matchIfMissing = true) +@ConditionalOnProperty(value = "app.auto-clear.exec-log.enabled", havingValue = "true") public class ExecLogFileAutoClearTask { /** * 分布式锁名称 */ - private static final String LOCK_KEY = "clear:elf:lock"; + private static final String LOCK_KEY = "clear:exl:lock"; @Resource - private AppExecLogConfig appExecLogConfig; - - @Keep - @Resource - private FileClient logsFileClient; + private AppExecLogAutoClearConfig appExecLogAutoClearConfig; @Resource - private ExecHostLogDAO execHostLogDAO; + private ExecLogService execLogService; /** * 清理 @@ -52,45 +44,23 @@ public class ExecLogFileAutoClearTask { public void clear() { log.info("ExecLogFileAutoClearTask.clear start"); // 获取锁并执行 - LockerUtils.tryLock(LOCK_KEY, this::doClearFile); + LockerUtils.tryLock(LOCK_KEY, this::doClear); log.info("ExecLogFileAutoClearTask.clear finish"); } /** - * 执行清理文件 + * 执行清理 */ - private void doClearFile() { + private void doClear() { // 删除的时间区间 - String maxPeriod = Dates.stream() - .subDay(appExecLogConfig.getKeepPeriod()) - .format(); - // 获取需要删除的最大id - ExecHostLogDO hostLog = execHostLogDAO.of() - .createWrapper() - .select(ExecHostLogDO::getLogId, ExecHostLogDO::getLogPath) - .lt(ExecHostLogDO::getCreateTime, maxPeriod) - .orderByDesc(ExecHostLogDO::getId) - .then() - .getOne(); - if (hostLog == null) { - return; - } - // 获取执行日志根目录 - String hostLogPath = logsFileClient.getAbsolutePath(hostLog.getLogPath()); - String execLogPath = Files1.getParentPath(hostLogPath); - String parentPath = Files1.getParentPath(execLogPath); - // 获取需要删除的文件 - List files = Files1.listFilesFilter(parentPath, s -> { - if (!Strings.isInteger(s.getName())) { - return false; - } - return Long.parseLong(s.getName()) <= hostLog.getLogId(); - }, false, true); - if (files.isEmpty()) { - return; - } - // 删除日志文件 - files.forEach(Files1::delete); + Date createLessEq = Dates.stream() + .subDay(appExecLogAutoClearConfig.getKeepPeriod()) + .date(); + // 清理 + ExecLogQueryRequest request = new ExecLogQueryRequest(); + request.setCreateTimeLe(createLessEq); + request.setStatusList(ExecStatusEnum.FINISH_STATUS_LIST); + execLogService.clearExecLog(request); } } diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/HostConnectLogAutoClearTask.java b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/HostConnectLogAutoClearTask.java new file mode 100644 index 00000000..3edd504c --- /dev/null +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/java/com/orion/visor/module/asset/task/HostConnectLogAutoClearTask.java @@ -0,0 +1,66 @@ +package com.orion.visor.module.asset.task; + +import com.orion.lang.utils.time.Dates; +import com.orion.visor.framework.common.utils.LockerUtils; +import com.orion.visor.module.asset.define.config.AppHostConnectLogAutoClearConfig; +import com.orion.visor.module.asset.entity.request.host.HostConnectLogQueryRequest; +import com.orion.visor.module.asset.enums.HostConnectStatusEnum; +import com.orion.visor.module.asset.service.HostConnectLogService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Date; + +/** + * 主机连接日志自动清理 + * + * @author Jiahang Li + * @version 1.0.0 + * @since 2024/4/24 23:50 + */ +@Slf4j +@Component +@ConditionalOnProperty(value = "app.auto-clear.host-connect-log.enabled", havingValue = "true") +public class HostConnectLogAutoClearTask { + + /** + * 分布式锁名称 + */ + private static final String LOCK_KEY = "clear:hcl:lock"; + + @Resource + private AppHostConnectLogAutoClearConfig appHostConnectLogAutoClearConfig; + + @Resource + private HostConnectLogService hostConnectLogService; + + /** + * 清理 + */ + @Scheduled(cron = "0 10 3 * * ?") + public void clear() { + log.info("HostConnectLogAutoClearTask.clear start"); + // 获取锁并执行 + LockerUtils.tryLock(LOCK_KEY, this::doClear); + log.info("HostConnectLogAutoClearTask.clear finish"); + } + + /** + * 执行清理 + */ + private void doClear() { + // 删除的时间区间 + Date createLessEq = Dates.stream() + .subDay(appHostConnectLogAutoClearConfig.getKeepPeriod()) + .date(); + // 清理 + HostConnectLogQueryRequest request = new HostConnectLogQueryRequest(); + request.setCreateTimeLe(createLessEq); + request.setStatusList(HostConnectStatusEnum.FINISH_STATUS_LIST); + hostConnectLogService.clearHostConnectLog(request); + } + +} diff --git a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/resources/META-INF/additional-spring-configuration-metadata.json b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/resources/META-INF/additional-spring-configuration-metadata.json index aba7302d..211c787c 100644 --- a/orion-visor-module-asset/orion-visor-module-asset-service/src/main/resources/META-INF/additional-spring-configuration-metadata.json +++ b/orion-visor-module-asset/orion-visor-module-asset-service/src/main/resources/META-INF/additional-spring-configuration-metadata.json @@ -19,6 +19,11 @@ "name": "app.auto-clear.exec-log", "type": "com.orion.visor.module.asset.define.config.AppExecLogAutoClearConfig", "sourceType": "com.orion.visor.module.asset.define.config.AppExecLogAutoClearConfig" + }, + { + "name": "app.auto-clear.host-connect-log", + "type": "com.orion.visor.module.asset.define.config.AppHostConnectLogAutoClearConfig", + "sourceType": "com.orion.visor.module.asset.define.config.AppHostConnectLogAutoClearConfig" } ], "properties": [ @@ -61,12 +66,22 @@ { "name": "app.auto-clear.exec-log.enabled", "type": "java.lang.Boolean", - "description": "开启 批量执行日志文件自动清理." + "description": "开启 批量执行日志自动清理." }, { "name": "app.auto-clear.exec-log.keep-period", "type": "java.lang.Integer", - "description": "批量执行日志文件自动清理 保留周期 (天)." + "description": "批量执行日志自动清理 保留周期 (天)." + }, + { + "name": "app.auto-clear.host-connect-log.enabled", + "type": "java.lang.Boolean", + "description": "开启 主机连接日志自动清理." + }, + { + "name": "app.auto-clear.host-connect-log.keep-period", + "type": "java.lang.Integer", + "description": "主机连接日志自动清理 保留周期 (天)." } ] } \ No newline at end of file