manager/orion-visor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ 批量上传优化.

Browse Source
This commit is contained in:
lijiahangmax
2024-05-11 00:16:42 +08:00
parent 0a43e5db45
commit 978d94dddf
21 changed files with 274 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/controller/UploadTaskController.java
View File

@@ -11,6 +11,7 @@ import com.orion.ops.module.asset.entity.request.upload.UploadTaskCreateRequest;
import com.orion.ops.module.asset.entity.request.upload.UploadTaskQueryRequest;
import com.orion.ops.module.asset.entity.request.upload.UploadTaskRequest;
import com.orion.ops.module.asset.entity.vo.UploadTaskCreateVO;
import com.orion.ops.module.asset.entity.vo.UploadTaskStatusVO;
import com.orion.ops.module.asset.entity.vo.UploadTaskVO;
import com.orion.ops.module.asset.service.UploadTaskService;
import io.swagger.v3.oas.annotations.Operation;
@@ -40,8 +41,6 @@ import java.util.List;
@SuppressWarnings({"ELValidationInJSP", "SpringElInspection"})
public class UploadTaskController {
// TODO 前端日志 测试删除慢吗
@Resource
private UploadTaskService uploadTaskService;
@@ -92,7 +91,7 @@ public class UploadTaskController {
@Operation(summary = "查询上传状态")
@Parameter(name = "id", description = "id", required = true)
@PreAuthorize("@ss.hasPermission('asset:upload-task:query')")
public List<UploadTaskVO> getUploadTaskStatus(@RequestParam("idList") List<Long> idList, @RequestParam("queryFiles") Boolean queryFiles) {
public List<UploadTaskStatusVO> getUploadTaskStatus(@RequestParam("idList") List<Long> idList, @RequestParam("queryFiles") Boolean queryFiles) {
return uploadTaskService.getUploadTaskStatus(idList, queryFiles);
}

5
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/convert/UploadTaskConvert.java
View File

@@ -3,6 +3,7 @@ package com.orion.ops.module.asset.convert;
import com.orion.ops.module.asset.entity.domain.UploadTaskDO;
import com.orion.ops.module.asset.entity.request.upload.UploadTaskCreateRequest;
import com.orion.ops.module.asset.entity.request.upload.UploadTaskQueryRequest;
import com.orion.ops.module.asset.entity.vo.UploadTaskStatusVO;
import com.orion.ops.module.asset.entity.vo.UploadTaskVO;
import org.mapstruct.Mapper;
import org.mapstruct.factory.Mappers;
@@ -27,6 +28,8 @@ public interface UploadTaskConvert {
UploadTaskVO to(UploadTaskDO domain);
List<UploadTaskVO> to(List<UploadTaskDO> list);
List<UploadTaskVO> toList(List<UploadTaskDO> list);
UploadTaskStatusVO toStatus(UploadTaskDO domain);
}

8
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/entity/domain/UploadTaskDO.java
View File

@@ -56,6 +56,14 @@ public class UploadTaskDO extends BaseDO {
@TableField("extra_info")
private String extraInfo;
@Schema(description = "文件数量")
@TableField("file_count")
private Integer fileCount;
@Schema(description = "主机数量")
@TableField("host_count")
private Integer hostCount;
@Schema(description = "开始时间")
@TableField("start_time")
private Date startTime;

3
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/entity/dto/HostTerminalConnectDTO.java
View File

@@ -38,6 +38,9 @@ public class HostTerminalConnectDTO {
@Schema(description = "主机地址")
private String hostAddress;
@Schema(description = "系统类型")
private String osType;
@Schema(description = "端口")
private Integer port;

44
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/entity/vo/UploadTaskStatusVO.java Normal file
View File

@@ -0,0 +1,44 @@
package com.orion.ops.module.asset.entity.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
/**
* 上传任务状态 视图响应对象
*
* @author Jiahang Li
* @version 1.0.7
* @since 2024-5-7 22:15
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Schema(name = "UploadTaskStatusVO", description = "上传任务状态 视图响应对象")
public class UploadTaskStatusVO implements Serializable {
private static final long serialVersionUID = 1L;
@Schema(description = "id")
private Long id;
@Schema(description = "状态")
private String status;
@Schema(description = "开始时间")
private Date startTime;
@Schema(description = "结束时间")
private Date endTime;
@Schema(description = "上传文件")
private List<UploadTaskFileVO> files;
}

6
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/entity/vo/UploadTaskVO.java
View File

@@ -47,6 +47,12 @@ public class UploadTaskVO implements Serializable {
@Schema(description = "额外信息")
private String extraInfo;
@Schema(description = "文件数量")
private Integer fileCount;
@Schema(description = "主机数量")
private Integer hostCount;
@Schema(description = "开始时间")
private Date startTime;

3
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/handler/host/upload/task/FileUploadTask.java
View File

@@ -1,6 +1,7 @@
package com.orion.ops.module.asset.handler.host.upload.task;
import com.orion.lang.utils.Threads;
import com.orion.lang.utils.io.Files1;
import com.orion.lang.utils.io.Streams;
import com.orion.ops.framework.common.constant.Const;
import com.orion.ops.module.asset.dao.UploadTaskDAO;
@@ -136,7 +137,7 @@ public class FileUploadTask implements IFileUploadTask {
.map(s -> FileUploadFileItemDTO.builder()
.id(s.getId())
.fileId(s.getFileId())
.remotePath(record.getRemotePath() + Const.SLASH + s.getFilePath())
.remotePath(Files1.getPath(Const.SLASH + record.getRemotePath() + Const.SLASH + s.getFilePath()))
.status(UploadTaskFileStatusEnum.WAITING.name())
.current(0L)
.build())

31
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/handler/host/upload/uploader/FileUploader.java
View File

@@ -1,15 +1,19 @@
package com.orion.ops.module.asset.handler.host.upload.uploader;
import com.orion.lang.utils.Strings;
import com.orion.lang.utils.collect.Maps;
import com.orion.lang.utils.io.Files1;
import com.orion.lang.utils.io.Streams;
import com.orion.net.host.SessionStore;
import com.orion.net.host.sftp.SftpExecutor;
import com.orion.ops.framework.common.constant.Const;
import com.orion.ops.framework.common.file.FileClient;
import com.orion.ops.framework.common.utils.PathUtils;
import com.orion.ops.module.asset.dao.UploadTaskFileDAO;
import com.orion.ops.module.asset.define.config.AppSftpConfig;
import com.orion.ops.module.asset.entity.domain.UploadTaskFileDO;
import com.orion.ops.module.asset.entity.dto.HostTerminalConnectDTO;
import com.orion.ops.module.asset.enums.HostSshOsTypeEnum;
import com.orion.ops.module.asset.enums.UploadTaskFileStatusEnum;
import com.orion.ops.module.asset.handler.host.upload.dto.FileUploadFileItemDTO;
import com.orion.ops.module.asset.service.HostTerminalService;
@@ -23,6 +27,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
@@ -100,9 +105,10 @@ public class FileUploader implements IFileUploader {
private boolean initSession() {
log.info("HostFileUploader.initSession start taskId: {}, hostId: {}", taskId, hostId);
try {
// TODO 测试看看有没有问题, 则修改为 打开 executor 后 是否会connect, 不需要的话就关闭 executor 然后重新打开
// 打开会话
// 替换用户路径
HostTerminalConnectDTO connectInfo = hostTerminalService.getTerminalConnectInfo(hostId);
this.replaceRemotePathVariable(connectInfo.getOsType(), connectInfo.getUsername());
// 打开会话
this.sessionStore = hostTerminalService.openSessionStore(connectInfo);
this.executor = sessionStore.getSftpExecutor(connectInfo.getFileNameCharset());
executor.connect();
@@ -215,6 +221,27 @@ public class FileUploader implements IFileUploader {
uploadTaskFileDAO.updateById(update);
}
/**
* 替换文件路径变量
*
* @param osType osType
* @param username username
*/
private void replaceRemotePathVariable(String osType, String username) {
// 包含变量
if (!files.get(0).getRemotePath().contains(Const.DOLLAR)) {
return;
}
String home = PathUtils.getHomePath(HostSshOsTypeEnum.WINDOWS.name().equals(osType), username);
// 替换变量
Map<String, String> env = Maps.newMap(4);
env.put("username", username);
env.put("home", home);
for (FileUploadFileItemDTO file : files) {
file.setRemotePath(Files1.getPath(Strings.format(file.getRemotePath(), env)));
}
}
@Override
public void cancel() {
log.info("HostFileUploader.cancel taskId: {}, hostId: {}, canceled: {}, closed: {}", taskId, hostId, canceled, closed);

3
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/service/UploadTaskService.java
View File

@@ -5,6 +5,7 @@ import com.orion.ops.module.asset.entity.request.upload.UploadTaskCreateRequest;
import com.orion.ops.module.asset.entity.request.upload.UploadTaskQueryRequest;
import com.orion.ops.module.asset.entity.request.upload.UploadTaskRequest;
import com.orion.ops.module.asset.entity.vo.UploadTaskCreateVO;
import com.orion.ops.module.asset.entity.vo.UploadTaskStatusVO;
import com.orion.ops.module.asset.entity.vo.UploadTaskVO;
import java.util.List;
@@ -51,7 +52,7 @@ public interface UploadTaskService {
* @param queryFiles queryFiles
* @return rows
*/
List<UploadTaskVO> getUploadTaskStatus(List<Long> idList, Boolean queryFiles);
List<UploadTaskStatusVO> getUploadTaskStatus(List<Long> idList, Boolean queryFiles);
/**
* 获取上传任务数量

1
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/service/impl/HostTerminalServiceImpl.java
View File

@@ -241,6 +241,7 @@ public class HostTerminalServiceImpl implements HostTerminalService {
conn.setHostId(host.getId());
conn.setHostName(host.getName());
conn.setHostAddress(host.getAddress());
conn.setOsType(config.getOsType());
conn.setPort(config.getPort());
conn.setTimeout(config.getConnectTimeout());
conn.setCharset(config.getCharset());

31
orion-ops-module-asset/orion-ops-module-asset-service/src/main/java/com/orion/ops/module/asset/service/impl/UploadTaskServiceImpl.java
View File

@@ -112,6 +112,8 @@ public class UploadTaskServiceImpl implements UploadTaskService {
record.setUsername(user.getUsername());
record.setDescription(Strings.def(record.getDescription(), () -> Strings.format(DEFAULT_DESC, Dates.current())));
record.setStatus(UploadTaskStatusEnum.WAITING.name());
record.setFileCount(files.size());
record.setHostCount(hostIdList.size());
UploadTaskExtraDTO extra = UploadTaskExtraDTO.builder()
.hostIdList(hostIdList)
.hosts(hosts)
@@ -159,7 +161,7 @@ public class UploadTaskServiceImpl implements UploadTaskService {
// 计算传输进度
this.computeUploadProgress(id, files);
// 设置任务文件
this.setTaskFiles(uploadTask, files);
this.setTaskHostFiles(uploadTask, files);
return uploadTask;
}
@@ -174,40 +176,39 @@ public class UploadTaskServiceImpl implements UploadTaskService {
}
@Override
public List<UploadTaskVO> getUploadTaskStatus(List<Long> idList, Boolean queryFiles) {
public List<UploadTaskStatusVO> getUploadTaskStatus(List<Long> idList, Boolean queryFiles) {
// 查询任务
List<UploadTaskVO> tasks = uploadTaskDAO.of()
List<UploadTaskStatusVO> tasks = uploadTaskDAO.of()
.createWrapper()
.select(UploadTaskDO::getId, UploadTaskDO::getStatus,
UploadTaskDO::getStartTime, UploadTaskDO::getEndTime)
.in(UploadTaskDO::getId, idList)
.then()
.list(UploadTaskConvert.MAPPER::to);
.list(UploadTaskConvert.MAPPER::toStatus);
if (!Booleans.isTrue(queryFiles)) {
return tasks;
}
// 查询任务文件
Map<Long, List<UploadTaskFileVO>> filesMap = uploadTaskFileDAO.of()
Map<Long, List<UploadTaskFileVO>> taskFilesMap = uploadTaskFileDAO.of()
.createWrapper()
.select(UploadTaskFileDO::getId, UploadTaskFileDO::getTaskId,
UploadTaskFileDO::getHostId, UploadTaskFileDO::getStatus,
.select(UploadTaskFileDO::getId, UploadTaskFileDO::getTaskId, UploadTaskFileDO::getHostId,
UploadTaskFileDO::getStatus, UploadTaskFileDO::getFileSize,
UploadTaskFileDO::getStartTime, UploadTaskFileDO::getEndTime)
.in(UploadTaskFileDO::getTaskId, idList)
.then()
.stream()
.map(UploadTaskFileConvert.MAPPER::to)
.collect(Collectors.groupingBy(UploadTaskFileVO::getTaskId));
for (UploadTaskVO task : tasks) {
for (UploadTaskStatusVO task : tasks) {
Long id = task.getId();
List<UploadTaskFileVO> files = filesMap.get(id);
List<UploadTaskFileVO> files = taskFilesMap.get(id);
if (files == null) {
files = Lists.empty();
} else {
// 计算进度
this.computeUploadProgress(id, files);
// 设置任务文件
}
this.setTaskFiles(task, files);
task.setFiles(files);
}
return tasks;
}
@@ -311,8 +312,8 @@ public class UploadTaskServiceImpl implements UploadTaskService {
return uploadTaskDAO.wrapper()
.eq(UploadTaskDO::getId, request.getId())
.eq(UploadTaskDO::getUserId, request.getUserId())
.in(UploadTaskDO::getDescription, request.getDescription())
.eq(UploadTaskDO::getRemotePath, request.getRemotePath())
.like(UploadTaskDO::getDescription, request.getDescription())
.like(UploadTaskDO::getRemotePath, request.getRemotePath())
.eq(UploadTaskDO::getStatus, request.getStatus())
.ge(UploadTaskDO::getCreateTime, Arrays1.getIfPresent(request.getCreateTimeRange(), 0))
.le(UploadTaskDO::getCreateTime, Arrays1.getIfPresent(request.getCreateTimeRange(), 1))
@@ -450,12 +451,12 @@ public class UploadTaskServiceImpl implements UploadTaskService {
}
/**
* 设置任务文件
* 设置主机任务文件
*
* @param task task
* @param files files
*/
private void setTaskFiles(UploadTaskVO task, List<UploadTaskFileVO> files) {
private void setTaskHostFiles(UploadTaskVO task, List<UploadTaskFileVO> files) {
Map<Long, List<UploadTaskFileVO>> hostFiles = files.stream()
.collect(Collectors.groupingBy(UploadTaskFileVO::getHostId));
List<UploadTaskHostVO> hosts = JSON.parseObject(task.getExtraInfo(), UploadTaskExtraDTO.class)

4
orion-ops-module-asset/orion-ops-module-asset-service/src/main/resources/mapper/UploadTaskMapper.xml
View File

@@ -11,6 +11,8 @@
<result column="description" property="description"/>
<result column="status" property="status"/>
<result column="extra_info" property="extraInfo"/>
<result column="file_count" property="fileCount"/>
<result column="host_count" property="hostCount"/>
<result column="start_time" property="startTime"/>
<result column="end_time" property="endTime"/>
<result column="create_time" property="createTime"/>
@@ -22,7 +24,7 @@
<!-- 通用查询结果列 -->
<sql id="Base_Column_List">
id, user_id, username, remote_path, description, status, extra_info, start_time, end_time, create_time, update_time, creator, updater, deleted
id, user_id, username, remote_path, description, status, extra_info, file_count, host_count, start_time, end_time, create_time, update_time, creator, updater, deleted
</sql>
</mapper>