🔨 优化执行日志查看逻辑.

This commit is contained in:
lijiahang
2025-02-05 10:16:12 +08:00
parent 89f6d2cd1c
commit 972103841c
9 changed files with 222 additions and 211 deletions

View File

@@ -22,13 +22,11 @@
*/
package org.dromara.visor.module.asset.handler.host.exec.log;
import cn.orionsec.kit.lang.annotation.Keep;
import cn.orionsec.kit.lang.utils.Strings;
import lombok.extern.slf4j.Slf4j;
import org.dromara.visor.common.constant.ExtraFieldConst;
import org.dromara.visor.common.interfaces.FileClient;
import org.dromara.visor.framework.websocket.core.utils.WebSockets;
import org.dromara.visor.module.asset.define.AssetThreadPools;
import org.dromara.visor.module.asset.entity.dto.ExecHostLogTailDTO;
import org.dromara.visor.module.asset.entity.dto.ExecLogTailDTO;
import org.dromara.visor.module.asset.handler.host.exec.log.constant.LogConst;
import org.dromara.visor.module.asset.handler.host.exec.log.manager.ExecLogManager;
@@ -52,41 +50,27 @@ import javax.annotation.Resource;
@Component
public class ExecLogTailHandler extends AbstractWebSocketHandler {
@Keep
@Resource
private FileClient logsFileClient;
@Resource
private ExecLogManager execLogManager;
@Override
public void afterConnectionEstablished(WebSocketSession session) {
protected void handleTextMessage(WebSocketSession session, TextMessage message) {
String id = session.getId();
log.info("ExecLogTailHandler-afterConnectionEstablished id: {}", id);
// 获取参数
ExecLogTailDTO info = WebSockets.getAttr(session, ExtraFieldConst.INFO);
// 打开会话
for (ExecHostLogTailDTO host : info.getHosts()) {
String trackerId = this.getTrackerId(id, info, host);
String absolutePath = logsFileClient.getAbsolutePath(host.getPath());
// 追踪器
ExecLogTracker tracker = new ExecLogTracker(trackerId,
absolutePath,
WebSockets.createSyncSession(session),
host);
// 执行
String payload = message.getPayload();
if (LogConst.PING_PAYLOAD.equals(payload)) {
// ping
WebSockets.sendText(session, LogConst.PONG_PAYLOAD);
} else if (Strings.isInteger(payload)) {
// 获取日志
ExecLogTailDTO info = WebSockets.getAttr(session, ExtraFieldConst.INFO);
Long execHostId = Long.valueOf(payload);
ExecLogTracker tracker = new ExecLogTracker(info.getExecId(),
execHostId,
WebSockets.createSyncSession(session));
// 执行追踪器
AssetThreadPools.EXEC_LOG.execute(tracker);
// 添加追踪器
execLogManager.addTracker(tracker);
}
}
@Override
protected void handleTextMessage(WebSocketSession session, TextMessage message) {
String payload = message.getPayload();
// ping
if (LogConst.PING_PAYLOAD.equals(payload)) {
WebSockets.sendText(session, LogConst.PONG_PAYLOAD);
execLogManager.addTracker(id, tracker);
}
}
@@ -99,24 +83,8 @@ public class ExecLogTailHandler extends AbstractWebSocketHandler {
public void afterConnectionClosed(WebSocketSession session, CloseStatus status) {
String id = session.getId();
log.info("ExecLogTailHandler-afterConnectionClosed id: {}, code: {}, reason: {}", id, status.getCode(), status.getReason());
// 关闭会话
ExecLogTailDTO info = WebSockets.getAttr(session, ExtraFieldConst.INFO);
// 移除追踪器
for (ExecHostLogTailDTO host : info.getHosts()) {
execLogManager.removeTracker(this.getTrackerId(id, info, host));
}
}
/**
* 获取追踪器 id
*
* @param id id
* @param info info
* @param host host
* @return trackerId
*/
private String getTrackerId(String id, ExecLogTailDTO info, ExecHostLogTailDTO host) {
return id + "_" + info.getId() + "_" + host.getId();
execLogManager.removeTrackers(id);
}
}

View File

@@ -28,6 +28,8 @@ import org.dromara.visor.common.constant.Const;
import org.dromara.visor.module.asset.handler.host.exec.log.tracker.IExecLogTracker;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@@ -43,61 +45,58 @@ import java.util.stream.Collectors;
@Component
public class ExecLogManager {
private final ConcurrentHashMap<String, IExecLogTracker> execTrackers = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, List<IExecLogTracker>> execTrackers = new ConcurrentHashMap<>();
/**
* 添加执行日志追踪器
*
* @param id id
* @param tracker tracker
*/
public void addTracker(IExecLogTracker tracker) {
execTrackers.put(tracker.getTrackerId(), tracker);
}
/**
* 获取日志追踪器
*
* @param trackerId trackerId
* @return tracker
*/
public IExecLogTracker getTracker(String trackerId) {
return execTrackers.get(trackerId);
public void addTracker(String id, IExecLogTracker tracker) {
execTrackers.computeIfAbsent(id, k -> new ArrayList<>()).add(tracker);
}
/**
* 移除日志追踪器
*
* @param trackerId trackerId
* @param id id
*/
public void removeTracker(String trackerId) {
IExecLogTracker tracker = execTrackers.remove(trackerId);
if (tracker != null) {
tracker.close();
public void removeTrackers(String id) {
// 移除并且关闭
List<IExecLogTracker> trackers = execTrackers.remove(id);
if (trackers != null) {
trackers.forEach(IExecLogTracker::close);
}
}
/**
* 异步关闭进行中的追踪器
*
* @param path path
* @param execHostId execHostId
*/
public void asyncCloseTailFile(String path) {
if (path == null) {
public void asyncCloseTailFile(Long execHostId) {
if (execHostId == null) {
return;
}
// 获取当前路径的全部追踪器
List<IExecLogTracker> trackers = execTrackers.values()
.stream()
.flatMap(Collection::stream)
.filter(s -> s.getExecHostId().equals(execHostId))
.collect(Collectors.toList());
if (trackers.isEmpty()) {
return;
}
// 异步设置更新并且关闭
Threads.start(() -> {
try {
// 获取当前路径的全部追踪器
List<IExecLogTracker> trackers = execTrackers.values()
.stream()
.filter(s -> s.getPath().equals(path))
.collect(Collectors.toList());
Threads.sleep(Const.MS_S_1);
trackers.forEach(IExecLogTracker::setLastModify);
Threads.sleep(Const.MS_S_5);
trackers.forEach(IExecLogTracker::close);
} catch (Exception e) {
log.error("ExecLogManager.asyncCloseTailFile error path: {}", path, e);
log.error("ExecLogManager.asyncCloseTailFile error execHostId: {}", execHostId, e);
}
});
}

View File

@@ -26,16 +26,32 @@ import cn.orionsec.kit.ext.tail.Tracker;
import cn.orionsec.kit.ext.tail.delay.DelayTrackerListener;
import cn.orionsec.kit.ext.tail.mode.FileNotFoundMode;
import cn.orionsec.kit.ext.tail.mode.FileOffsetMode;
import cn.orionsec.kit.lang.exception.argument.InvalidArgumentException;
import cn.orionsec.kit.lang.utils.Charsets;
import cn.orionsec.kit.lang.utils.io.FileReaders;
import cn.orionsec.kit.lang.utils.io.Files1;
import cn.orionsec.kit.lang.utils.io.Streams;
import cn.orionsec.kit.spring.SpringHolder;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.dromara.visor.common.constant.Const;
import org.dromara.visor.common.constant.ErrorMessage;
import org.dromara.visor.common.interfaces.FileClient;
import org.dromara.visor.common.utils.Valid;
import org.dromara.visor.framework.websocket.core.utils.WebSockets;
import org.dromara.visor.module.asset.dao.ExecHostLogDAO;
import org.dromara.visor.module.asset.define.config.AppLogConfig;
import org.dromara.visor.module.asset.entity.dto.ExecHostLogTailDTO;
import org.dromara.visor.module.asset.entity.domain.ExecHostLogDO;
import org.dromara.visor.module.asset.enums.ExecHostStatusEnum;
import org.dromara.visor.module.asset.handler.host.exec.log.constant.LogConst;
import org.springframework.web.socket.WebSocketSession;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.Charset;
/**
* log tracker 实现类
*
@@ -48,76 +64,166 @@ public class ExecLogTracker implements IExecLogTracker {
private static final AppLogConfig appLogConfig = SpringHolder.getBean(AppLogConfig.class);
private static final FileClient localFileClient = SpringHolder.getBean("localFileClient");
private static final ExecHostLogDAO execHostLogDAO = SpringHolder.getBean(ExecHostLogDAO.class);
private final WebSocketSession session;
private final ExecHostLogTailDTO config;
@Getter
private final Long execId;
@Getter
private final String trackerId;
private final Long execHostId;
@Getter
private final String absolutePath;
private Charset charset;
private String absolutePath;
private ExecHostLogDO execHostLog;
private RandomAccessFile file;
private DelayTrackerListener tracker;
private volatile boolean close;
public ExecLogTracker(String trackerId,
String absolutePath,
WebSocketSession session,
ExecHostLogTailDTO config) {
this.trackerId = trackerId;
this.absolutePath = absolutePath;
public ExecLogTracker(Long execId,
Long execHostId,
WebSocketSession session) {
this.execId = execId;
this.execHostId = execHostId;
this.session = session;
this.config = config;
}
@Override
public void run() {
try {
this.tracker = new DelayTrackerListener(absolutePath, this);
tracker.charset(config.getCharset());
tracker.delayMillis(appLogConfig.getTrackerDelay());
tracker.offset(FileOffsetMode.LINE, appLogConfig.getTrackerOffset());
tracker.notFoundMode(FileNotFoundMode.WAIT_COUNT, Const.N_10);
// 开始监听文件
tracker.run();
// 初始化数据
this.initData();
// 查看日志
if (ExecHostStatusEnum.RUNNING.name().equals(execHostLog.getStatus())) {
// 追踪文件
this.tailFile();
} else {
// 直接读取文件
this.readFile();
}
} catch (InvalidArgumentException e) {
// 业务异常
log.error("exec log tracker init error id: {}", execHostId, e);
// 发送消息
this.sendMessage(e.getMessage());
} catch (Exception e) {
log.error("exec log tracker error path: {}", absolutePath, e);
log.error("exec log tracker exec error id: {}", execHostId, e);
} finally {
// 释放资源
this.close();
}
}
@Override
public void setLastModify() {
tracker.setFileLastModifyTime();
/**
* 初始化数据
*/
private void initData() {
// 读取数据
this.execHostLog = execHostLogDAO.selectByIdAndLogId(execHostId, execId);
Valid.notNull(execHostLog, ErrorMessage.DATA_ABSENT);
// 检查任务状态
Valid.neq(execHostLog.getStatus(), ExecHostStatusEnum.WAITING.name(), ErrorMessage.ILLEGAL_STATUS);
// 获取文件路径
this.absolutePath = localFileClient.getAbsolutePath(execHostLog.getLogPath());
Valid.isTrue(Files1.isFile(absolutePath), ErrorMessage.FILE_ABSENT);
// 获取编码集
this.charset = Charsets.of(this.getCharset());
}
@Override
public String getPath() {
return config.getPath();
/**
* 追踪文件
*/
private void tailFile() {
// 创建追踪器
this.tracker = new DelayTrackerListener(absolutePath, this);
tracker.charset(charset.name());
tracker.delayMillis(appLogConfig.getTrackerLoadInterval());
tracker.offset(FileOffsetMode.LINE, appLogConfig.getTrackerLoadLines());
tracker.notFoundMode(FileNotFoundMode.WAIT_COUNT, Const.N_10);
// 开始追踪
tracker.run();
}
@Override
public void read(byte[] bytes, int len, Tracker tracker) {
// 发送消息
String message = config.getId() + LogConst.SEPARATOR + new String(bytes, 0, len);
/**
* 读取文件
*
* @throws IOException IOException
*/
private void readFile() throws IOException {
this.file = Files1.openRandomAccess(absolutePath, Const.ACCESS_R);
// 获取文件位置
long pos = FileReaders.readTailLinesSeek(file, appLogConfig.getTrackerLoadLines());
// 设置文件位置
file.seek(pos);
// 读取到尾部
byte[] buffer = new byte[Const.BUFFER_KB_8];
int len;
while ((len = file.read(buffer)) != -1) {
// 发送消息
this.sendMessage(new String(buffer, 0, len, charset));
}
}
/**
* 读取参数中的编码集
*
* @return charset
*/
private String getCharset() {
JSONObject params = JSON.parseObject(execHostLog.getParameter());
if (params != null) {
String charset = params.getString(Const.CHARSET);
if (charset != null) {
return charset;
}
}
return Const.UTF_8;
}
/**
* 发送消息
*
* @param message message
*/
private void sendMessage(String message) {
try {
WebSockets.sendText(session, message);
WebSockets.sendText(session, execHostId + LogConst.SEPARATOR + message);
} catch (Exception e) {
log.error("ExecLogTracker.send error", e);
}
}
@Override
public void setLastModify() {
if (tracker != null) {
tracker.setFileLastModifyTime();
}
}
@Override
public void read(byte[] bytes, int len, Tracker tracker) {
// 发送消息
this.sendMessage(new String(bytes, 0, len, charset));
}
@Override
public void close() {
log.info("ExecLogTracker.close path: {}, closed: {}", absolutePath, close);
log.info("ExecLogTracker.close execHostId: {}, closed: {}", execHostId, close);
if (close) {
return;
}
this.close = true;
if (file != null) {
Streams.close(file);
}
if (tracker != null) {
tracker.stop();
}

View File

@@ -34,30 +34,23 @@ import cn.orionsec.kit.lang.able.SafeCloseable;
*/
public interface IExecLogTracker extends Runnable, DataHandler, SafeCloseable {
/**
* 获取 execId
*
* @return execId
*/
Long getExecId();
/**
* 获取 execHostId
*
* @return execHostId
*/
Long getExecHostId();
/**
* 设置最后修改时间
*/
void setLastModify();
/**
* 获取 id
*
* @return id
*/
String getTrackerId();
/**
* 获取路径
*
* @return path
*/
String getPath();
/**
* 获取绝对路径
*
* @return 绝对路径
*/
String getAbsolutePath();
}

View File

@@ -28,7 +28,6 @@ import org.dromara.visor.module.asset.entity.domain.ExecLogDO;
import org.dromara.visor.module.asset.entity.dto.ExecLogTailDTO;
import org.dromara.visor.module.asset.entity.request.exec.ExecLogClearRequest;
import org.dromara.visor.module.asset.entity.request.exec.ExecLogQueryRequest;
import org.dromara.visor.module.asset.entity.request.exec.ExecLogTailRequest;
import org.dromara.visor.module.asset.entity.vo.ExecLogStatusVO;
import org.dromara.visor.module.asset.entity.vo.ExecLogVO;
@@ -139,10 +138,10 @@ public interface ExecLogService {
/**
* 查看执行日志
*
* @param request request
* @param id id
* @return token
*/
String getExecLogTailToken(ExecLogTailRequest request);
String getExecLogTailToken(Long id);
/**
* 获取查看执行日志参数

View File

@@ -24,10 +24,6 @@ package org.dromara.visor.module.asset.service;
import org.dromara.visor.common.handler.data.model.GenericsDataModel;
import org.dromara.visor.module.asset.entity.domain.HostDO;
import org.dromara.visor.module.asset.enums.HostTypeEnum;
import java.util.List;
import java.util.Map;
/**
* 主机配置 服务类
@@ -36,47 +32,24 @@ import java.util.Map;
* @version 1.0.0
* @since 2023-9-11 14:16
*/
// TODO 待优化
public interface HostConfigService {
/**
* 获取主机配置
*
* @param id id
* @param type type
* @param <T> T
* @param id id
* @param <T> T
* @return host
*/
<T extends GenericsDataModel> T getHostConfig(Long id, HostTypeEnum type);
/**
* 构建主机配置
*
* @param host host
* @param type type
* @param <T> T
* @return host
*/
<T extends GenericsDataModel> T buildHostConfig(HostDO host, HostTypeEnum type);
<T extends GenericsDataModel> T getHostConfig(Long id);
/**
* 获取主机配置
*
* @param idList idList
* @param type type
* @param <T> T
* @return config
* @param host host
* @param <T> T
* @return host
*/
<T extends GenericsDataModel> Map<Long, T> getHostConfigMap(List<Long> idList, HostTypeEnum type);
/**
* 构建主机配置
*
* @param hostList hostList
* @param type type
* @param <T> T
* @return config
*/
<T extends GenericsDataModel> Map<Long, T> buildHostConfigMap(List<HostDO> hostList, HostTypeEnum type);
<T extends GenericsDataModel> T getHostConfig(HostDO host);
}

View File

@@ -22,7 +22,6 @@
*/
package org.dromara.visor.module.asset.service.impl;
import cn.orionsec.kit.lang.function.Functions;
import lombok.extern.slf4j.Slf4j;
import org.dromara.visor.common.constant.ErrorMessage;
import org.dromara.visor.common.handler.data.model.GenericsDataModel;
@@ -31,16 +30,10 @@ import org.dromara.visor.module.asset.dao.HostDAO;
import org.dromara.visor.module.asset.entity.domain.HostDO;
import org.dromara.visor.module.asset.enums.HostStatusEnum;
import org.dromara.visor.module.asset.enums.HostTypeEnum;
import org.dromara.visor.module.asset.handler.host.config.model.HostSshConfigModel;
import org.dromara.visor.module.asset.service.HostConfigService;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* 主机配置 服务实现类
@@ -57,48 +50,23 @@ public class HostConfigServiceImpl implements HostConfigService {
private HostDAO hostDAO;
@Override
public <T extends GenericsDataModel> T getHostConfig(Long id, HostTypeEnum type) {
public <T extends GenericsDataModel> T getHostConfig(Long id) {
// 查询主机
HostDO host = hostDAO.selectById(id);
// 转换为配置
return this.buildHostConfig(host, type);
return this.getHostConfig(host);
}
@Override
@SuppressWarnings("unchecked")
public <T extends GenericsDataModel> T buildHostConfig(HostDO host, HostTypeEnum type) {
public <T extends GenericsDataModel> T getHostConfig(HostDO host) {
Valid.notNull(host, ErrorMessage.HOST_ABSENT);
// 检查主机类型
Valid.isTrue(type.name().equals(host.getType()), ErrorMessage.HOST_TYPE_ERROR);
HostTypeEnum type = HostTypeEnum.of(host.getType());
// 检查主机状态
Valid.isTrue(HostStatusEnum.ENABLED.name().equals(host.getStatus()), ErrorMessage.HOST_NOT_ENABLED);
// 查询主机配置
HostSshConfigModel model = type.parse(host.getConfig());
Valid.notNull(model, ErrorMessage.CONFIG_ABSENT);
return (T) model;
}
@Override
public <T extends GenericsDataModel> Map<Long, T> getHostConfigMap(List<Long> idList, HostTypeEnum type) {
// 查询主机
Map<Long, HostDO> hostMap = hostDAO.selectBatchIds(idList)
.stream()
.collect(Collectors.toMap(HostDO::getId, Function.identity(), Functions.right()));
// 转换为配置
Map<Long, T> result = new HashMap<>();
for (Long id : idList) {
result.put(id, this.buildHostConfig(hostMap.get(id), type));
}
return result;
}
@Override
public <T extends GenericsDataModel> Map<Long, T> buildHostConfigMap(List<HostDO> hostList, HostTypeEnum type) {
Map<Long, T> result = new HashMap<>();
for (HostDO host : hostList) {
result.put(host.getId(), this.buildHostConfig(host, type));
}
return result;
T config = type.parse(host.getConfig());
Valid.notNull(config, ErrorMessage.CONFIG_ABSENT);
return (T) config;
}
}

View File

@@ -42,7 +42,10 @@ import org.dromara.visor.module.asset.entity.dto.TerminalAccessDTO;
import org.dromara.visor.module.asset.entity.dto.TerminalConnectDTO;
import org.dromara.visor.module.asset.entity.dto.TerminalTransferDTO;
import org.dromara.visor.module.asset.entity.vo.TerminalThemeVO;
import org.dromara.visor.module.asset.enums.*;
import org.dromara.visor.module.asset.enums.HostExtraItemEnum;
import org.dromara.visor.module.asset.enums.HostExtraSshAuthTypeEnum;
import org.dromara.visor.module.asset.enums.HostIdentityTypeEnum;
import org.dromara.visor.module.asset.enums.HostSshAuthTypeEnum;
import org.dromara.visor.module.asset.handler.host.config.model.HostSshConfigModel;
import org.dromara.visor.module.asset.handler.host.extra.model.HostSshExtraModel;
import org.dromara.visor.module.asset.service.HostConfigService;
@@ -171,7 +174,7 @@ public class TerminalServiceImpl implements TerminalService {
// 查询主机
HostDO host = hostDAO.selectById(hostId);
// 查询主机配置
HostSshConfigModel config = hostConfigService.buildHostConfig(host, HostTypeEnum.SSH);
HostSshConfigModel config = hostConfigService.getHostConfig(host);
// 获取配置
return this.getHostConnectInfo(host, config, null);
}
@@ -195,7 +198,7 @@ public class TerminalServiceImpl implements TerminalService {
ErrorMessage.ANY_NO_PERMISSION,
DataPermissionTypeEnum.HOST_GROUP.getPermissionName());
// 获取主机配置
HostSshConfigModel config = hostConfigService.buildHostConfig(host, HostTypeEnum.SSH);
HostSshConfigModel config = hostConfigService.getHostConfig(host);
Valid.notNull(config, ErrorMessage.CONFIG_ABSENT);
// 查询主机额外配置
HostSshExtraModel extra = hostExtraService.getHostExtra(userId, hostId, HostExtraItemEnum.SSH);

View File

@@ -26,6 +26,7 @@ import cn.orionsec.kit.lang.utils.Booleans;
import cn.orionsec.kit.lang.utils.Strings;
import cn.orionsec.kit.net.host.sftp.SftpExecutor;
import cn.orionsec.kit.net.host.sftp.SftpFile;
import cn.orionsec.kit.spring.SpringHolder;
import com.alibaba.fastjson.JSON;
import org.dromara.visor.module.asset.define.config.AppSftpConfig;
import org.dromara.visor.module.asset.handler.host.transfer.model.SftpFileBackupParams;
@@ -39,19 +40,20 @@ import org.dromara.visor.module.asset.handler.host.transfer.model.SftpFileBackup
*/
public class SftpUtils {
private static final AppSftpConfig appSftpConfig = SpringHolder.getBean(AppSftpConfig.class);
private SftpUtils() {
}
/**
* 检查上传文件是否存在 并且执行响应策略
*
* @param config config
* @param executor executor
* @param path path
*/
public static void checkUploadFilePresent(AppSftpConfig config, SftpExecutor executor, String path) {
public static void checkUploadFilePresent(SftpExecutor executor, String path) {
// 重复不备份
if (!Booleans.isTrue(config.getUploadPresentBackup())) {
if (!Booleans.isTrue(appSftpConfig.getUploadPresentBackup())) {
return;
}
// 检查文件是否存在
@@ -59,7 +61,7 @@ public class SftpUtils {
if (file != null) {
// 文件存在则备份
SftpFileBackupParams backupParams = new SftpFileBackupParams(file.getName());
String target = Strings.format(config.getBackupFileName(), JSON.parseObject(JSON.toJSONString(backupParams)));
String target = Strings.format(appSftpConfig.getUploadBackupFileName(), JSON.parseObject(JSON.toJSONString(backupParams)));
// 移动
executor.move(path, target);
}