diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/DashboardController.java b/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/DashboardController.java index e9d28c9c..f69eac29 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/DashboardController.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/DashboardController.java @@ -2,6 +2,7 @@ import DGU_AI_LAB.admin_be.domain.dashboard.controller.docs.DashBoardApi; import DGU_AI_LAB.admin_be.domain.dashboard.service.DashboardService; +import DGU_AI_LAB.admin_be.domain.monitoring.service.MyGpuUsageService; import DGU_AI_LAB.admin_be.domain.requests.dto.response.UserServerResponseDTO; import DGU_AI_LAB.admin_be.domain.requests.entity.StatusFilter; import DGU_AI_LAB.admin_be.global.auth.CustomUserDetails; @@ -11,6 +12,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @@ -24,6 +26,7 @@ public class DashboardController implements DashBoardApi { private final DashboardService dashboardService; + private final MyGpuUsageService myGpuUsageService; /** * 사용자 신청 현황 서버 목록 조회 API @@ -40,4 +43,13 @@ public ResponseEntity> getUserServers( return SuccessResponse.ok(userServers); } -} \ No newline at end of file + /** 내 서버의 GPU별 사용률·메모리·내 프로세스. 다른 사용자는 사람 수와 메모리 합계만 돌려준다. */ + @Override + public ResponseEntity> getMyGpuUsage( + @AuthenticationPrincipal CustomUserDetails principal, + @PathVariable Long requestId + ) { + return SuccessResponse.ok(myGpuUsageService.getMyGpuUsage(requestId, principal.getUserId())); + } + +} diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/docs/DashBoardApi.java b/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/docs/DashBoardApi.java index b268a082..f96a00f7 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/docs/DashBoardApi.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/docs/DashBoardApi.java @@ -1,5 +1,6 @@ package DGU_AI_LAB.admin_be.domain.dashboard.controller.docs; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.MyGpuUsageResponseDTO; import DGU_AI_LAB.admin_be.domain.requests.dto.response.UserServerResponseDTO; import DGU_AI_LAB.admin_be.domain.requests.entity.StatusFilter; import DGU_AI_LAB.admin_be.error.dto.ErrorResponse; @@ -14,6 +15,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -34,4 +36,25 @@ ResponseEntity> getUserServers( @Parameter(description = "조회할 서버 신청 상태 필터 (PENDING, FULFILLED, DENIED, DELETED, ALL)", required = true, example = "FULFILLED") @RequestParam(name = "status") StatusFilter statusFilter ); + + @Operation(summary = "내 서버 GPU 사용 현황 조회", + description = "내 서버(FULFILLED 신청)의 GPU별 사용률·메모리와 내 프로세스를 조회합니다. 다른 사용자는 사람 수와 메모리 합계로만 담기며 식별 정보는 없습니다. " + + "프로세스 정보를 읽지 못하면 processesAvailable=false, GPU 지표를 읽지 못하면 502입니다.") + @ApiResponse(responseCode = "200", description = "성공", + content = @Content(schema = @Schema(implementation = MyGpuUsageResponseDTO.class))) + @ApiResponse(responseCode = "401", description = "인증되지 않은 사용자", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @ApiResponse(responseCode = "403", description = "다른 사용자의 신청", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @ApiResponse(responseCode = "404", description = "없는 신청이거나 실행 중(FULFILLED)이 아닌 신청", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @ApiResponse(responseCode = "502", description = "GPU 지표 조회 실패", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @GetMapping("/me/servers/{requestId}/gpus") + ResponseEntity> getMyGpuUsage( + @Parameter(hidden = true) + @AuthenticationPrincipal CustomUserDetails principal, + @Parameter(description = "신청 ID", required = true, example = "1") + @PathVariable Long requestId + ); } \ No newline at end of file diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClient.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClient.java index 8d7ff196..21f0b12e 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClient.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClient.java @@ -32,39 +32,54 @@ public PrometheusClient(@Value("${prometheus.base-url}") String baseUrl) { this.webClient = webClient; } + /** 시계열 하나의 라벨 전체와 수치. */ + public record Sample(Map labels, double value) { + } + /** * 결과 벡터를 labelKey 값 → 수치로 돌려준다. 해당 라벨이 없는 시계열은 "unknown"으로 묶는다. - * 값이 없거나 수로 읽히지 않는 시계열은 건너뛴다 — 하나 때문에 나머지 결과까지 버리지 않는다. */ - @SuppressWarnings("unchecked") public Map queryVector(String promql, String labelKey) { + Map out = new LinkedHashMap<>(); + for (Sample s : querySeries(promql)) { + out.put(s.labels().getOrDefault(labelKey, "unknown"), s.value()); + } + return out; + } + + /** + * 결과 벡터를 라벨 전체와 함께 돌려준다. 값이 없거나 수로 읽히지 않는 시계열은 건너뛴다 — + * 하나 때문에 나머지 결과까지 버리지 않는다. + */ + @SuppressWarnings("unchecked") + public List querySeries(String promql) { Map response = webClient.get() .uri(uriBuilder -> uriBuilder .path("/api/v1/query") - .queryParam("query", promql) - .build()) + // PromQL의 {Hostname=~"…"} 같은 중괄호가 URI 템플릿 변수로 읽히지 않게 값은 변수로 넘긴다. + .queryParam("query", "{promql}") + .build(promql)) .retrieve() .bodyToMono(Map.class) .timeout(TIMEOUT) .block(); - if (response == null) return Map.of(); + if (response == null) return List.of(); Map data = (Map) response.get("data"); - if (data == null) return Map.of(); + if (data == null) return List.of(); List> results = (List>) data.get("result"); - if (results == null) return Map.of(); + if (results == null) return List.of(); - Map out = new LinkedHashMap<>(); + List out = new java.util.ArrayList<>(); for (Map item : results) { Map metric = (Map) item.get("metric"); List value = (List) item.get("value"); - String key = metric == null ? "unknown" : metric.getOrDefault(labelKey, "unknown"); Double parsed = parseSample(value); if (parsed == null) { - log.warn("[PrometheusClient] 읽을 수 없는 시계열을 건너뜀: label={}, value={}", key, value); + log.warn("[PrometheusClient] 읽을 수 없는 시계열을 건너뜀: labels={}, value={}", metric, value); continue; } - out.put(key, parsed); + out.add(new Sample(metric == null ? Map.of() : metric, parsed)); } return out; } @@ -73,7 +88,9 @@ public Map queryVector(String promql, String labelKey) { private static Double parseSample(List value) { if (value == null || value.size() < 2 || value.get(1) == null) return null; try { - return Double.valueOf(value.get(1).toString()); + double v = Double.parseDouble(value.get(1).toString()); + // Prometheus는 "NaN"·"+Inf"도 문자열로 돌려준다. 비유한 값은 수치로 쓸 수 없다. + return Double.isFinite(v) ? v : null; } catch (NumberFormatException e) { return null; } diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/AdminGpuUsageController.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/AdminGpuUsageController.java new file mode 100644 index 00000000..b6730fb9 --- /dev/null +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/AdminGpuUsageController.java @@ -0,0 +1,22 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.controller; + +import DGU_AI_LAB.admin_be.domain.monitoring.controller.docs.AdminGpuUsageApi; +import DGU_AI_LAB.admin_be.domain.monitoring.service.AdminGpuUsageService; +import DGU_AI_LAB.admin_be.global.common.SuccessResponse; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/** 관리자 전용(/api/admin/** 은 SecurityConfig에서 ADMIN만 허용). */ +@RestController +@RequiredArgsConstructor +public class AdminGpuUsageController implements AdminGpuUsageApi { + + private final AdminGpuUsageService adminGpuUsageService; + + @Override + public ResponseEntity> getGpuUsage(@RequestParam(name = "node", required = false) String node) { + return SuccessResponse.ok(adminGpuUsageService.getGpuUsage(node)); + } +} diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/docs/AdminGpuUsageApi.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/docs/AdminGpuUsageApi.java new file mode 100644 index 00000000..21e2e534 --- /dev/null +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/docs/AdminGpuUsageApi.java @@ -0,0 +1,41 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.controller.docs; + +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.AdminGpuUsageResponseDTO; +import DGU_AI_LAB.admin_be.error.dto.ErrorResponse; +import DGU_AI_LAB.admin_be.global.common.SuccessResponse; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; + +@Tag(name = "9. 관리자 GPU 사용 현황", description = "관리자용 노드별 GPU 사용률·프로세스·소유자 조회 API") +@RequestMapping("/api/admin/gpu-usage") +public interface AdminGpuUsageApi { + + @Operation(summary = "노드별 GPU 사용 현황 조회", + description = "GPU별 사용률·메모리와 그 위에서 도는 프로세스(소유자 이름·계정·신청 번호 포함)를 노드별로 조회합니다. " + + "node를 생략하면 DB의 모든 노드입니다. exporter가 없으면 processesAvailable=false이고 프로세스는 비어 있습니다. " + + "소유자를 못 찾은 프로세스는 ownerName 등이 null입니다.") + @ApiResponse(responseCode = "200", description = "성공", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = AdminGpuUsageResponseDTO.class)))) + @ApiResponse(responseCode = "401", description = "인증되지 않은 사용자", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @ApiResponse(responseCode = "403", description = "관리자가 아님", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @ApiResponse(responseCode = "404", description = "DB에 없는 노드", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @ApiResponse(responseCode = "502", description = "노드 하나를 지정했는데 GPU 지표 조회에 실패", + content = @Content(schema = @Schema(implementation = ErrorResponse.class))) + @GetMapping + ResponseEntity> getGpuUsage( + @Parameter(description = "노드 이름(생략하면 전체)", example = "farm6") + @RequestParam(name = "node", required = false) String node + ); +} diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/AdminGpuUsageResponseDTO.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/AdminGpuUsageResponseDTO.java new file mode 100644 index 00000000..9a9c7eb8 --- /dev/null +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/AdminGpuUsageResponseDTO.java @@ -0,0 +1,40 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.dto.response; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Builder; + +import java.util.List; + +/** + * 관리자용 노드별 GPU 사용 현황. 사용자 응답(MyGpuUsageResponseDTO)과 달리 프로세스마다 소유자를 담는다 — 관리자 전용 경로에서만 쓴다. + */ +@Builder +@Schema(description = "관리자용 노드 하나의 GPU 사용 현황") +public record AdminGpuUsageResponseDTO( + @Schema(description = "노드 이름", example = "farm6") String nodeName, + @Schema(description = "프로세스 정보를 읽을 수 있는지. false면 processes는 비어 있고 '없음'이 아니라 '모름'이다") boolean processesAvailable, + @Schema(description = "GPU별 현황") List gpus) { + + @Builder + public record Gpu( + @Schema(description = "GPU 번호", example = "0") int index, + @Schema(description = "GPU UUID") String uuid, + @Schema(description = "GPU 모델명") String model, + @Schema(description = "GPU 전체 사용률(%)") double utilizationPercent, + @Schema(description = "GPU 전체 사용 메모리(MiB)") double memoryUsedMiB, + @Schema(description = "GPU 전체 메모리(MiB)") double memoryTotalMiB, + @Schema(description = "이 GPU에서 도는 프로세스(메모리 큰 순)") List processes) { + } + + @Builder + public record Process( + @Schema(description = "소유자 이름. 소유자를 못 찾았으면 null(미확인)", nullable = true) String ownerName, + @Schema(description = "소유자 우분투 계정. 미확인이면 null", nullable = true) String ubuntuUsername, + @Schema(description = "소유자 사용자 번호. 미확인이거나 옛 Docker 컨테이너면 null", nullable = true) Long userId, + @Schema(description = "소유 신청 번호. 미확인이거나 옛 Docker 컨테이너면 null", nullable = true) Long requestId, + @Schema(description = "컨테이너 ID 앞 12자") String containerId, + @Schema(description = "프로세스 이름(파일 이름만)") String processName, + @Schema(description = "GPU 메모리(MiB)") double memoryUsedMiB, + @Schema(description = "실행된 지 지난 시간(초)") long elapsedSeconds) { + } +} diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/MyGpuUsageResponseDTO.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/MyGpuUsageResponseDTO.java new file mode 100644 index 00000000..9c0eec30 --- /dev/null +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/dto/response/MyGpuUsageResponseDTO.java @@ -0,0 +1,38 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.dto.response; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Builder; + +import java.util.List; + +/** + * 내 서버의 GPU별 사용 현황. 다른 사용자는 사람 수와 메모리 합계로만 담는다 — + * 이름·계정·Pod·프로세스 등 식별 정보는 이 응답에 넣지 않는다. + */ +@Builder +@Schema(description = "내 서버의 GPU별 사용 현황") +public record MyGpuUsageResponseDTO( + @Schema(description = "서버(노드) 이름", example = "farm1") String nodeName, + @Schema(description = "GPU별 현황") List gpus, + @Schema(description = "프로세스·다른 사용자 정보를 읽었는지. false면 내 프로세스 없음·다른 사용자 0명이 '없음'이 아니라 '모름'이다") + boolean processesAvailable) { + + @Builder + public record Gpu( + @Schema(description = "GPU 번호", example = "0") int index, + @Schema(description = "GPU 모델명") String modelName, + @Schema(description = "GPU 전체 사용률(%)") double utilizationPercent, + @Schema(description = "GPU 전체 사용 메모리(MiB)") double memoryUsedMiB, + @Schema(description = "GPU 전체 메모리(MiB)") double memoryTotalMiB, + @Schema(description = "내 프로세스(이름·실행 시간·GPU 메모리)") List myProcesses, + @Schema(description = "같은 GPU를 쓰는 다른 사용자 수") int otherUserCount, + @Schema(description = "다른 사용자들의 GPU 메모리 합계(MiB)") double otherMemoryMiB) { + } + + @Builder + public record Process( + @Schema(description = "프로세스 이름") String name, + @Schema(description = "실행된 지 지난 시간(초)") long elapsedSeconds, + @Schema(description = "GPU 메모리(MiB)") double memoryMiB) { + } +} diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageService.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageService.java new file mode 100644 index 00000000..c0a878fe --- /dev/null +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageService.java @@ -0,0 +1,103 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.service; + +import DGU_AI_LAB.admin_be.domain.monitoring.client.PrometheusClient.Sample; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.AdminGpuUsageResponseDTO; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.AdminGpuUsageResponseDTO.Gpu; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.AdminGpuUsageResponseDTO.Process; +import DGU_AI_LAB.admin_be.domain.monitoring.service.MyGpuUsageService.Owner; +import DGU_AI_LAB.admin_be.domain.monitoring.service.MyGpuUsageService.Snapshot; +import DGU_AI_LAB.admin_be.domain.nodes.entity.Node; +import DGU_AI_LAB.admin_be.domain.nodes.repository.NodeRepository; +import DGU_AI_LAB.admin_be.error.ErrorCode; +import DGU_AI_LAB.admin_be.error.exception.BusinessException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 관리자용 노드별 GPU 사용 현황: GPU마다 프로세스와 소유자를 담는다. + * Prometheus 조회·소유자 해석·짧은 캐시는 MyGpuUsageService의 것을 그대로 쓴다. + * 사용자 응답과 달리 노드에 꽂힌 GPU를 모두 보인다(관리자 화면이라 거르지 않는다). + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class AdminGpuUsageService { + + private final MyGpuUsageService myGpuUsageService; + private final NodeRepository nodeRepository; + + /** + * @param node null이면 DB의 모든 노드, 아니면 그 노드 하나(DB에 없으면 404). + * 노드 하나를 지정했는데 GPU 지표를 못 읽으면 502, 전체 조회에서는 그 노드만 빈 GPU·processesAvailable=false로 둔다. + */ + public List getGpuUsage(String node) { + if (node != null && !node.isBlank()) { + if (!nodeRepository.existsById(node)) throw new BusinessException(ErrorCode.RESOURCE_NOT_FOUND); + return List.of(usageOf(node, true)); + } + return nodeRepository.findAll().stream() + .map(Node::getNodeId) + .sorted() + .map(id -> usageOf(id, false)) + .toList(); + } + + private AdminGpuUsageResponseDTO usageOf(String node, boolean failHard) { + Snapshot snap; + try { + snap = myGpuUsageService.snapshot(node); + } catch (Exception e) { + log.warn("[ADMIN-GPU] GPU 지표 조회 실패: node={} {}", node, e.toString()); + if (failHard) throw new BusinessException(ErrorCode.EXTERNAL_API_ERROR); + return AdminGpuUsageResponseDTO.builder().nodeName(node).processesAvailable(false).gpus(List.of()).build(); + } + + Map> byGpu = new HashMap<>(); + if (snap.processesAvailable()) { + for (Sample s : snap.procMem()) { + String uuid = s.labels().get("gpu_uuid"); + String containerId = s.labels().get("container_id"); + Owner owner = MyGpuUsageService.ownerOf(snap, containerId, uuid); + byGpu.computeIfAbsent(uuid, k -> new ArrayList<>()).add(Process.builder() + .ownerName(owner == null ? null : owner.name()) + .ubuntuUsername(owner == null ? null : owner.ubuntuUsername()) + .userId(owner == null ? null : owner.userId()) + .requestId(owner == null ? null : owner.requestId()) + .containerId(containerId == null || containerId.isBlank() ? null : MyGpuUsageService.shortId(containerId)) + .processName(MyGpuUsageService.baseName(s.labels().get("process_name"))) + .memoryUsedMiB(s.value() / MyGpuUsageService.MIB) + .elapsedSeconds(snap.elapsed().getOrDefault(MyGpuUsageService.procKey(s), 0.0).longValue()) + .build()); + } + } + + List gpus = snap.util().stream() + .filter(s -> s.labels().get("UUID") != null) + .map(s -> { + String uuid = s.labels().get("UUID"); + double used = snap.used().containsKey(uuid) ? snap.used().get(uuid).value() : 0; + double free = snap.free().containsKey(uuid) ? snap.free().get(uuid).value() : 0; + List procs = new ArrayList<>(byGpu.getOrDefault(uuid, List.of())); + procs.sort(Comparator.comparingDouble(Process::memoryUsedMiB).reversed()); + return Gpu.builder() + .index(MyGpuUsageService.parseIndex(s.labels().get("gpu"))) + .uuid(uuid) + .model(s.labels().getOrDefault("modelName", "")) + .utilizationPercent(s.value()) + .memoryUsedMiB(used) + .memoryTotalMiB(used + free) + .processes(procs) + .build(); + }) + .sorted(Comparator.comparingInt(Gpu::index)) + .toList(); + return AdminGpuUsageResponseDTO.builder().nodeName(node).processesAvailable(snap.processesAvailable()).gpus(gpus).build(); + } +} diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageService.java b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageService.java new file mode 100644 index 00000000..dad5b42d --- /dev/null +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageService.java @@ -0,0 +1,337 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.service; + +import DGU_AI_LAB.admin_be.domain.monitoring.client.PrometheusClient; +import DGU_AI_LAB.admin_be.domain.monitoring.client.PrometheusClient.Sample; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.MyGpuUsageResponseDTO; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.MyGpuUsageResponseDTO.Gpu; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.MyGpuUsageResponseDTO.Process; +import DGU_AI_LAB.admin_be.domain.nodes.entity.Node; +import DGU_AI_LAB.admin_be.domain.nodes.repository.NodeRepository; +import DGU_AI_LAB.admin_be.domain.requests.entity.Request; +import DGU_AI_LAB.admin_be.domain.requests.entity.Status; +import DGU_AI_LAB.admin_be.domain.requests.repository.RequestRepository; +import DGU_AI_LAB.admin_be.error.ErrorCode; +import DGU_AI_LAB.admin_be.error.exception.BusinessException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +/** + * 내 서버(Pod가 올라간 노드)의 GPU별 사용률·메모리와 프로세스. 다른 사용자의 정보는 여기서 집계해 + * 사람 수·메모리 합계만 내보낸다 — 응답 DTO에 식별 정보를 담을 자리를 두지 않는다. + * + *

트랜잭션을 길게 잡지 않는다: DB 읽기는 저장소 호출마다 끝나고, Prometheus 호출은 DB 밖에서 한다. + * 노드별 Prometheus 결과는 짧게 캐시해 사용자가 화면을 열 때마다 질의가 나가지 않게 한다. + * + *

보여 주는 GPU: config-server가 Pod에 GPU를 줄 때 노드에 등록된 GPU 종류와 같은 모델만 UUID로 넘기므로 + * (NVIDIA_VISIBLE_DEVICES), 같은 규칙으로 그 모델의 GPU만 내 GPU로 본다. 한계: 노드가 DB에 없거나 모델 + * 목록이 비어 있으면 거를 근거가 없어 노드의 모든 GPU를 보인다. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class MyGpuUsageService { + + static final double MIB = 1024.0 * 1024.0; + static final long CACHE_TTL_MS = 10_000; + private static final ExecutorService POOL = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r, "my-gpu-prom"); + t.setDaemon(true); + return t; + }); + + private final PrometheusClient prometheusClient; + private final RequestRepository requestRepository; + private final NodeRepository nodeRepository; + + private final Map cache = new ConcurrentHashMap<>(); + private final Map locks = new ConcurrentHashMap<>(); + + /** 노드 하나의 Prometheus 결과와 소유자 해석. processesAvailable=false면 프로세스 쪽 조회가 실패한 것이다. */ + record Snapshot(List util, Map used, Map free, + List procMem, Map elapsed, + Map podOwnerByContainer, Map dockerOwnerByContainerGpu, + boolean processesAvailable) { + } + + /** + * 프로세스 소유자. key는 같은 사람을 Pod·옛 Docker에서 같게 세는 값(우분투 계정)이다. + * 옛 Docker 컨테이너는 userId·requestId를 알 수 없어 null이다. + */ + record Owner(String key, Long userId, String name, String ubuntuUsername, Long requestId) { + } + + private record Cached(Snapshot snapshot, long at) { + } + + public MyGpuUsageResponseDTO getMyGpuUsage(Long requestId, Long userId) { + Request request = requestRepository.findByIdWithUser(requestId) + .orElseThrow(() -> new BusinessException(ErrorCode.RESOURCE_NOT_FOUND)); + if (!userId.equals(request.getUser().getUserId())) { + throw new BusinessException(ErrorCode.FORBIDDEN); + } + // 삭제·회수 중인 신청은 podName/nodeName이 남아 있어도 그 노드의 사용량을 보여 주지 않는다. + if (request.getStatus() != Status.FULFILLED) { + throw new BusinessException(ErrorCode.RESOURCE_NOT_FOUND); + } + String node = request.getNodeName(); + if (request.getPodName() == null || node == null) { + return response(node, List.of(), true); + } + + Snapshot snap; + try { + snap = snapshot(node); + } catch (Exception e) { + // 'GPU 없음'과 구분되도록 빈 목록으로 숨기지 않고 오류로 알린다. + log.warn("[MY-GPU] GPU 지표 조회 실패: node={} {}", node, e.toString()); + throw new BusinessException(ErrorCode.EXTERNAL_API_ERROR); + } + if (snap.util().isEmpty()) { + log.warn("[MY-GPU] 노드 이름과 맞는 GPU 지표가 없음: node={} (DCGM Hostname 라벨과 노드 이름이 다를 수 있다)", node); + } + + Set modelsWanted = nodeModels(node); + String me = ownerKey(request); + Map> mine = new HashMap<>(); + Map otherMem = new HashMap<>(); + Map> otherUsers = new HashMap<>(); + for (Sample s : snap.procMem()) { + String uuid = s.labels().get("gpu_uuid"); + String containerId = s.labels().get("container_id"); + Owner ownerInfo = ownerOf(snap, containerId, uuid); + String owner = ownerInfo == null ? null : ownerInfo.key(); + double mib = s.value() / MIB; + if (me.equals(owner)) { + mine.computeIfAbsent(uuid, k -> new ArrayList<>()).add(new Process( + baseName(s.labels().get("process_name")), + snap.elapsed().getOrDefault(procKey(s), 0.0).longValue(), + mib)); + continue; + } + otherMem.merge(uuid, mib, Double::sum); + // 소유자를 못 찾은 컨테이너도 다른 사용자 한 명으로 센다. 컨테이너 ID가 없으면 PID로, + // 둘 다 없으면 사람 수에는 넣지 않고 메모리에만 더한다. + String who = owner != null ? owner + : isBlank(containerId) ? (isBlank(s.labels().get("pid")) ? null : "p:" + s.labels().get("pid")) + : "c:" + containerId; + if (who != null) otherUsers.computeIfAbsent(uuid, k -> new HashSet<>()).add(who); + } + + List gpus = snap.util().stream() + .filter(s -> s.labels().get("UUID") != null) + .filter(s -> modelsWanted.isEmpty() || modelsWanted.contains(normalizeModel(s.labels().get("modelName")))) + .map(s -> { + String uuid = s.labels().get("UUID"); + double usedMiB = snap.used().containsKey(uuid) ? snap.used().get(uuid).value() : 0; + double freeMiB = snap.free().containsKey(uuid) ? snap.free().get(uuid).value() : 0; + return Gpu.builder() + .index(parseIndex(s.labels().get("gpu"))) + .modelName(s.labels().getOrDefault("modelName", "")) + .utilizationPercent(s.value()) + .memoryUsedMiB(usedMiB) + .memoryTotalMiB(usedMiB + freeMiB) + .myProcesses(mine.getOrDefault(uuid, List.of())) + .otherUserCount(otherUsers.getOrDefault(uuid, Set.of()).size()) + .otherMemoryMiB(otherMem.getOrDefault(uuid, 0.0)) + .build(); + }) + .sorted(Comparator.comparingInt(Gpu::index)) + .toList(); + return response(node, gpus, snap.processesAvailable()); + } + + private static MyGpuUsageResponseDTO response(String node, List gpus, boolean processesAvailable) { + return MyGpuUsageResponseDTO.builder().nodeName(node).gpus(gpus).processesAvailable(processesAvailable).build(); + } + + Set nodeModels(String node) { + try { + return nodeRepository.findByIdWithGpus(node).map(Node::getGpuModels).orElse(List.of()).stream() + .map(MyGpuUsageService::normalizeModel).collect(Collectors.toSet()); + } catch (Exception e) { + log.warn("[MY-GPU] 노드 GPU 모델 조회 실패, 모든 GPU를 보임: node={} {}", node, e.toString()); + return Set.of(); + } + } + + /** config-server의 normalize_gpu_model과 같은 규칙: 제조사·제품군 낱말을 빼고 남은 낱말이 같으면 같은 모델. */ + static String normalizeModel(String name) { + return Arrays.stream((name == null ? "" : name).toUpperCase(Locale.ROOT).split("[^0-9A-Z]+")) + .filter(t -> !t.isEmpty() && !t.equals("NVIDIA") && !t.equals("GEFORCE") && !t.equals("GENERATION")) + .collect(Collectors.joining(" ")); + } + + static Owner ownerOf(Snapshot snap, String containerId, String gpuUuid) { + if (isBlank(containerId)) return null; + Owner pod = snap.podOwnerByContainer().get(containerId); + if (pod != null) return pod; + // 옛 Docker 컨테이너: 지표에 12자 단축 ID만 있어 같은 GPU에서 보이는 시계열일 때만 같은 컨테이너로 본다. + return snap.dockerOwnerByContainerGpu().get(shortId(containerId) + "/" + gpuUuid); + } + + static String shortId(String id) { + return id.length() > 12 ? id.substring(0, 12) : id; + } + + private static boolean isBlank(String s) { + return s == null || s.isBlank(); + } + + /** 노드별 짧은 캐시. 같은 노드를 동시에 부르면 한 번만 조회하고, 실패는 캐시하지 않는다. */ + Snapshot snapshot(String node) { + Cached c = cache.get(node); + if (c != null && System.currentTimeMillis() - c.at() < CACHE_TTL_MS) return c.snapshot(); + synchronized (locks.computeIfAbsent(node, k -> new Object())) { + c = cache.get(node); + if (c != null && System.currentTimeMillis() - c.at() < CACHE_TTL_MS) return c.snapshot(); + Snapshot fresh = load(node); + cache.put(node, new Cached(fresh, System.currentTimeMillis())); + return fresh; + } + } + + private Snapshot load(String node) { + String sel = hostSelector("Hostname", node); + CompletableFuture> utilF = CompletableFuture.supplyAsync( + () -> prometheusClient.querySeries("DCGM_FI_DEV_GPU_UTIL" + sel), POOL); + CompletableFuture> usedF = CompletableFuture.supplyAsync( + () -> prometheusClient.querySeries("DCGM_FI_DEV_FB_USED" + sel), POOL); + CompletableFuture> freeF = CompletableFuture.supplyAsync( + () -> prometheusClient.querySeries("DCGM_FI_DEV_FB_FREE" + sel), POOL); + List util = utilF.join(); + Map used = byUuid(usedF.join()); + Map free = byUuid(freeF.join()); + + List procMem = List.of(); + Map elapsed = new HashMap<>(); + Map podOwners = new HashMap<>(); + Map dockerOwners = new HashMap<>(); + boolean available = true; + try { + // exporter가 이 노드에서 프로세스 목록을 읽고 있는지: 없으면(옛 exporter·미배포) 또는 0이면 '프로세스 없음'이 아니라 '모름'이다. + boolean up = prometheusClient.querySeries("gpu_process_metrics_up" + hostSelector("hostname", node)) + .stream().anyMatch(s -> s.value() == 1.0); + if (!up) { + log.warn("[MY-GPU] 프로세스 지표 없음(gpu_process_metrics_up): node={}", node); + return new Snapshot(util, used, free, List.of(), elapsed, podOwners, dockerOwners, false); + } + String uuids = util.stream().map(s -> s.labels().get("UUID")).filter(u -> u != null && u.matches("[A-Za-z0-9-]+")) + .collect(Collectors.joining("|")); + if (!uuids.isEmpty()) { + String psel = "{gpu_uuid=~\"" + uuids + "\"}"; + CompletableFuture> memF = CompletableFuture.supplyAsync( + () -> prometheusClient.querySeries("gpu_process_memory_used_bytes" + psel), POOL); + CompletableFuture> elF = CompletableFuture.supplyAsync( + () -> prometheusClient.querySeries("gpu_process_elapsed_seconds" + psel), POOL); + procMem = memF.join(); + for (Sample s : elF.join()) elapsed.put(procKey(s), s.value()); + resolveOwners(procMem, podOwners, dockerOwners); + } + } catch (Exception e) { + // 프로세스 정보 없이도 GPU별 사용률은 보여 준다. 대신 '모름'임을 응답에 남긴다. + log.warn("[MY-GPU] 프로세스 지표 조회 실패: node={} {}", node, e.toString()); + procMem = List.of(); + available = false; + } + return new Snapshot(util, used, free, procMem, elapsed, podOwners, dockerOwners, available); + } + + /** + * DCGM Hostname 라벨은 노드 이름(예: farm6)이다. 대소문자·FQDN 차이는 허용한다. + * 노드 이름에 쓸 수 없는 문자가 있으면 정확히 같은 값만 찾는다. + */ + static String hostSelector(String label, String node) { + if (node.matches("[A-Za-z0-9.-]+")) { + return "{" + label + "=~\"(?i)" + node.replace(".", "[.]") + "([.].*)?\"}"; + } + return "{" + label + "=\"" + node.replace("\\", "\\\\").replace("\"", "\\\"") + "\"}"; + } + + /** + * Pod는 kube_pod_container_info의 container_id로 Pod 이름을 찾아 신청의 사용자로, + * 옛 Docker 컨테이너는 docker_gpu_user_container_info의 ubuntu_username으로 찾는다. + */ + private void resolveOwners(List procMem, Map podOwners, Map dockerOwners) { + Set ids = procMem.stream().map(s -> s.labels().get("container_id")) + .filter(id -> id != null && id.matches("[0-9a-f]{64}")).collect(Collectors.toSet()); + if (ids.isEmpty()) return; + + Map podByContainer = new HashMap<>(); + for (Sample s : prometheusClient.querySeries( + "kube_pod_container_info{container_id=~\".+://(" + String.join("|", ids) + ")\"}")) { + String cid = s.labels().get("container_id"); + String pod = s.labels().get("pod"); + if (cid != null && pod != null) podByContainer.put(cid.substring(cid.lastIndexOf('/') + 1), pod); + } + if (!podByContainer.isEmpty()) { + Map ownerByPod = new HashMap<>(); + for (Request r : requestRepository.findAllByPodNameIn(new HashSet<>(podByContainer.values()))) { + ownerByPod.put(r.getPodName(), new Owner(ownerKey(r), r.getUser().getUserId(), r.getUser().getName(), + r.getUser().getUbuntuUsername(), r.getRequestId())); + } + podByContainer.forEach((cid, pod) -> { + if (ownerByPod.containsKey(pod)) podOwners.put(cid, ownerByPod.get(pod)); + }); + } + + Set unresolved = ids.stream().filter(id -> !podOwners.containsKey(id)).collect(Collectors.toSet()); + if (unresolved.isEmpty()) return; + for (Sample s : prometheusClient.querySeries("docker_gpu_user_container_info{container_id=~\"" + + unresolved.stream().map(MyGpuUsageService::shortId).collect(Collectors.joining("|")) + "\"}")) { + String cid = s.labels().get("container_id"); + String uuid = s.labels().get("gpu_uuid"); + String user = s.labels().get("ubuntu_username"); + if (cid != null && uuid != null && !isBlank(user)) { + dockerOwners.put(cid + "/" + uuid, new Owner(user, null, s.labels().get("real_name"), user, null)); + } + } + } + + /** 같은 사용자는 Pod든 옛 Docker 컨테이너든 같은 키가 되도록 우분투 계정을 쓴다. */ + static String ownerKey(Request r) { + String name = r.getUser().getUbuntuUsername(); + return name != null && !name.isBlank() ? name : "id:" + r.getUser().getUserId(); + } + + static Map byUuid(List samples) { + Map m = new HashMap<>(); + for (Sample s : samples) { + String uuid = s.labels().get("UUID"); + if (uuid != null) m.put(uuid, s); + } + return m; + } + + static String procKey(Sample s) { + return s.labels().get("gpu_uuid") + "/" + s.labels().get("container_id") + "/" + s.labels().get("pid"); + } + + static String baseName(String path) { + if (path == null || path.isBlank()) return ""; + return path.substring(path.lastIndexOf('/') + 1); + } + + static int parseIndex(String gpu) { + try { + return Integer.parseInt(gpu); + } catch (NumberFormatException e) { + return Integer.MAX_VALUE; + } + } +} diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/nodes/repository/NodeRepository.java b/src/main/java/DGU_AI_LAB/admin_be/domain/nodes/repository/NodeRepository.java index baf5054d..1136a38d 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/nodes/repository/NodeRepository.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/nodes/repository/NodeRepository.java @@ -3,13 +3,19 @@ import DGU_AI_LAB.admin_be.domain.nodes.entity.Node; import DGU_AI_LAB.admin_be.domain.resourceGroups.entity.ResourceGroup; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import java.util.Collection; import java.util.List; +import java.util.Optional; @Repository public interface NodeRepository extends JpaRepository { + @Query("SELECT n FROM Node n LEFT JOIN FETCH n.gpus WHERE n.nodeId = :nodeId") + Optional findByIdWithGpus(@Param("nodeId") String nodeId); + List findAllByResourceGroup(ResourceGroup resourceGroup); List findAllByResourceGroupIn(Collection resourceGroups); } diff --git a/src/main/java/DGU_AI_LAB/admin_be/domain/requests/repository/RequestRepository.java b/src/main/java/DGU_AI_LAB/admin_be/domain/requests/repository/RequestRepository.java index fc4f144b..7af7e433 100644 --- a/src/main/java/DGU_AI_LAB/admin_be/domain/requests/repository/RequestRepository.java +++ b/src/main/java/DGU_AI_LAB/admin_be/domain/requests/repository/RequestRepository.java @@ -34,6 +34,11 @@ public interface RequestRepository extends JpaRepository { List findAllByUser_UserIdAndStatus(Long userId, Status status); boolean existsByPodName(String podName); + @Query("SELECT r FROM Request r JOIN FETCH r.user WHERE r.podName IN :podNames") + List findAllByPodNameIn(@Param("podNames") Collection podNames); + + @Query("SELECT r FROM Request r JOIN FETCH r.user WHERE r.requestId = :requestId") + Optional findByIdWithUser(@Param("requestId") Long requestId); boolean existsByUser_UbuntuUsernameAndStatusIn(String ubuntuUsername, Collection statuses); boolean existsByUser_UserIdAndStatus(Long userId, Status status); boolean existsByUser_UserIdAndStatusIn(Long userId, Collection statuses); diff --git a/src/test/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/DashboardControllerTest.java b/src/test/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/DashboardControllerTest.java new file mode 100644 index 00000000..8c562aa1 --- /dev/null +++ b/src/test/java/DGU_AI_LAB/admin_be/domain/dashboard/controller/DashboardControllerTest.java @@ -0,0 +1,98 @@ +package DGU_AI_LAB.admin_be.domain.dashboard.controller; + +import DGU_AI_LAB.admin_be.domain.dashboard.service.DashboardService; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.MyGpuUsageResponseDTO; +import DGU_AI_LAB.admin_be.domain.monitoring.service.MyGpuUsageService; +import DGU_AI_LAB.admin_be.domain.users.entity.User; +import DGU_AI_LAB.admin_be.error.ErrorCode; +import DGU_AI_LAB.admin_be.error.exception.BusinessException; +import DGU_AI_LAB.admin_be.global.auth.CustomUserDetails; +import DGU_AI_LAB.admin_be.support.WebMvcTestSupport; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.core.MethodParameter; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.web.bind.support.WebDataBinderFactory; +import org.springframework.web.context.request.NativeWebRequest; +import org.springframework.web.method.support.HandlerMethodArgumentResolver; +import org.springframework.web.method.support.ModelAndViewContainer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@WebMvcTest(value = DashboardController.class, excludeAutoConfiguration = SecurityAutoConfiguration.class) +@Import(DashboardControllerTest.PrincipalConfig.class) +class DashboardControllerTest extends WebMvcTestSupport { + + /** 보안 설정을 끈 슬라이스라 @AuthenticationPrincipal 대신 로그인한 사용자(id=1)를 직접 꽂는다. */ + @TestConfiguration + static class PrincipalConfig { + @Bean + WebMvcConfigurer principalResolver() { + return new WebMvcConfigurer() { + @Override + public void addArgumentResolvers(List resolvers) { + resolvers.add(new HandlerMethodArgumentResolver() { + @Override + public boolean supportsParameter(MethodParameter p) { + return p.getParameterType() == CustomUserDetails.class; + } + + @Override + public Object resolveArgument(MethodParameter p, ModelAndViewContainer m, NativeWebRequest r, + WebDataBinderFactory b) { + User user = mock(User.class); + when(user.getUserId()).thenReturn(1L); + return new CustomUserDetails(user); + } + }); + } + }; + } + } + + @Autowired private MockMvc mockMvc; + @MockitoBean private DashboardService dashboardService; + @MockitoBean private MyGpuUsageService myGpuUsageService; + + @Test + @DisplayName("GET /me/servers/{id}/gpus: 내 신청이면 200과 GPU 목록") + void ok() throws Exception { + when(myGpuUsageService.getMyGpuUsage(10L, 1L)).thenReturn( + MyGpuUsageResponseDTO.builder().nodeName("farm6").gpus(List.of()).processesAvailable(true).build()); + + mockMvc.perform(get("/api/dashboard/me/servers/10/gpus")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data.nodeName").value("farm6")) + .andExpect(jsonPath("$.data.processesAvailable").value(true)); + } + + @Test + @DisplayName("GET /me/servers/{id}/gpus: 남의 신청이면 403") + void forbidden() throws Exception { + when(myGpuUsageService.getMyGpuUsage(10L, 1L)).thenThrow(new BusinessException(ErrorCode.FORBIDDEN)); + + mockMvc.perform(get("/api/dashboard/me/servers/10/gpus")).andExpect(status().isForbidden()); + } + + @Test + @DisplayName("GET /me/servers/{id}/gpus: 없거나 실행 중이 아닌 신청이면 404") + void notFound() throws Exception { + when(myGpuUsageService.getMyGpuUsage(10L, 1L)).thenThrow(new BusinessException(ErrorCode.RESOURCE_NOT_FOUND)); + + mockMvc.perform(get("/api/dashboard/me/servers/10/gpus")).andExpect(status().isNotFound()); + } +} diff --git a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClientTest.java b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClientTest.java index 2a63e096..2e1dd101 100644 --- a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClientTest.java +++ b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/client/PrometheusClientTest.java @@ -53,6 +53,17 @@ void queryVector_mapsByLabel() { assertThat(captured.get().url().getQuery()).isEqualTo("query=avg by(Hostname)(X)"); } + @Test + @DisplayName("라벨 선택자의 중괄호가 URI 템플릿 변수로 읽히지 않고 인코딩돼 나간다") + void querySeries_braceSelector() { + PrometheusClient client = clientReturning("{\"status\":\"success\",\"data\":{\"result\":[]}}"); + + client.querySeries("DCGM_FI_DEV_GPU_UTIL{Hostname=~\"(?i)farm1([.].*)?\"}"); + + var query = captured.get().url().getRawQuery(); + assertThat(query).startsWith("query=DCGM_FI_DEV_GPU_UTIL%7BHostname"); + } + @Test @DisplayName("data나 result가 없으면 빈 결과를 돌려준다") void queryVector_emptyWhenNoData() { @@ -61,7 +72,7 @@ void queryVector_emptyWhenNoData() { } @Test - @DisplayName("값이 없거나 수로 읽히지 않는 시계열은 건너뛰고 나머지는 돌려준다") + @DisplayName("값이 없거나 수로 읽히지 않거나 유한하지 않은 시계열은 건너뛰고 나머지는 돌려준다") void queryVector_skipsMalformedSeries() { PrometheusClient client = clientReturning(""" {"status":"success","data":{"resultType":"vector","result":[ @@ -69,13 +80,14 @@ void queryVector_skipsMalformedSeries() { {"metric":{"Hostname":"no-value"}}, {"metric":{"Hostname":"short"},"value":[1700000000]}, {"metric":{"Hostname":"text"},"value":[1700000000,"+Inf"]}, - {"metric":{"Hostname":"node-b"},"value":[1700000000,"NaN"]} + {"metric":{"Hostname":"node-b"},"value":[1700000000,"NaN"]}, + {"metric":{"Hostname":"node-c"},"value":[1700000000,"-Inf"]} ]}} """); var result = client.queryVector("q", "Hostname"); - assertThat(result).containsOnlyKeys("node-a", "node-b"); + assertThat(result).containsOnlyKeys("node-a"); // NaN·±Inf는 수치로 쓸 수 없어 건너뛴다 assertThat(result.get("node-a")).isEqualTo(42.5); } } diff --git a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/AdminGpuUsageControllerTest.java b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/AdminGpuUsageControllerTest.java new file mode 100644 index 00000000..f9a1df75 --- /dev/null +++ b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/controller/AdminGpuUsageControllerTest.java @@ -0,0 +1,58 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.controller; + +import DGU_AI_LAB.admin_be.domain.monitoring.service.AdminGpuUsageService; +import DGU_AI_LAB.admin_be.support.WebMvcTestSupport; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; +import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.List; + +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.mockito.ArgumentMatchers.eq; + +@WebMvcTest( + value = AdminGpuUsageController.class, + excludeAutoConfiguration = {SecurityAutoConfiguration.class} +) +class AdminGpuUsageControllerTest extends WebMvcTestSupport { + + @Autowired + private MockMvc mockMvc; + + @MockitoBean + private AdminGpuUsageService adminGpuUsageService; + + @Test + @DisplayName("node를 생략하면 전체 노드(null)로 조회하고 200과 노드 목록을 준다") + void allNodesWhenNodeOmitted() throws Exception { + when(adminGpuUsageService.getGpuUsage(isNull())).thenReturn(List.of( + new DGU_AI_LAB.admin_be.domain.monitoring.dto.response.AdminGpuUsageResponseDTO( + "farm6", false, List.of()))); + + mockMvc.perform(get("/api/admin/gpu-usage")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data[0].nodeName").value("farm6")) + .andExpect(jsonPath("$.data[0].processesAvailable").value(false)); + } + + @Test + @DisplayName("node 파라미터를 서비스에 그대로 넘긴다") + void passesNodeParameter() throws Exception { + when(adminGpuUsageService.getGpuUsage(eq("farm6"))).thenReturn(List.of()); + + mockMvc.perform(get("/api/admin/gpu-usage").param("node", "farm6")) + .andExpect(status().isOk()); + + verify(adminGpuUsageService).getGpuUsage("farm6"); + } +} diff --git a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageServiceTest.java b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageServiceTest.java new file mode 100644 index 00000000..f9526742 --- /dev/null +++ b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/AdminGpuUsageServiceTest.java @@ -0,0 +1,162 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.service; + +import DGU_AI_LAB.admin_be.domain.monitoring.client.PrometheusClient.Sample; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.AdminGpuUsageResponseDTO; +import DGU_AI_LAB.admin_be.domain.monitoring.service.MyGpuUsageService.Owner; +import DGU_AI_LAB.admin_be.domain.monitoring.service.MyGpuUsageService.Snapshot; +import DGU_AI_LAB.admin_be.domain.nodes.entity.Node; +import DGU_AI_LAB.admin_be.domain.nodes.repository.NodeRepository; +import DGU_AI_LAB.admin_be.error.ErrorCode; +import DGU_AI_LAB.admin_be.error.exception.BusinessException; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class AdminGpuUsageServiceTest { + + @Mock MyGpuUsageService myGpuUsageService; + @Mock NodeRepository nodeRepository; + @InjectMocks AdminGpuUsageService service; + + private static final String C_KNOWN = "a".repeat(64); + private static final String C_UNKNOWN = "e".repeat(64); + + @org.junit.jupiter.api.BeforeEach + void nodeExistsByDefault() { + when(nodeRepository.existsById(anyString())).thenReturn(true); + } + + private static Sample s(double v, String... kv) { + Map m = new LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) m.put(kv[i], kv[i + 1]); + return new Sample(m, v); + } + + private static Snapshot snap(boolean available, List procMem, Map podOwners) { + List util = List.of(s(40, "UUID", "GPU-1", "gpu", "0", "modelName", "RTX 3090")); + return new Snapshot(util, + Map.of("GPU-1", s(10240, "UUID", "GPU-1")), + Map.of("GPU-1", s(14336, "UUID", "GPU-1")), + procMem, Map.of("GPU-1/" + C_KNOWN + "/11", 3600.0), podOwners, Map.of(), available); + } + + @Test + @DisplayName("프로세스 지표가 없으면 GPU 사용률·메모리만 주고 processesAvailable=false, 프로세스는 비운다") + void processesUnavailableKeepsGpuMetricsOnly() { + when(myGpuUsageService.snapshot("farm6")).thenReturn(snap(false, List.of(), Map.of())); + + AdminGpuUsageResponseDTO dto = service.getGpuUsage("farm6").get(0); + + assertThat(dto.nodeName()).isEqualTo("farm6"); + assertThat(dto.processesAvailable()).isFalse(); + assertThat(dto.gpus()).hasSize(1); + AdminGpuUsageResponseDTO.Gpu gpu = dto.gpus().get(0); + assertThat(gpu.uuid()).isEqualTo("GPU-1"); + assertThat(gpu.model()).isEqualTo("RTX 3090"); + assertThat(gpu.utilizationPercent()).isEqualTo(40); + assertThat(gpu.memoryUsedMiB()).isEqualTo(10240); + assertThat(gpu.memoryTotalMiB()).isEqualTo(24576); + assertThat(gpu.processes()).isEmpty(); + } + + @Test + @DisplayName("소유자를 못 찾은 프로세스는 소유자 필드가 전부 null(미확인)이고 나머지 정보는 채운다") + void unresolvedOwnerIsNull() { + Sample proc = s(2 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_UNKNOWN, "pid", "55", + "process_name", "/home/zed/train.py"); + when(myGpuUsageService.snapshot("farm6")).thenReturn(snap(true, List.of(proc), Map.of())); + + AdminGpuUsageResponseDTO.Process p = service.getGpuUsage("farm6").get(0).gpus().get(0).processes().get(0); + + assertThat(p.ownerName()).isNull(); + assertThat(p.ubuntuUsername()).isNull(); + assertThat(p.userId()).isNull(); + assertThat(p.requestId()).isNull(); + assertThat(p.containerId()).isEqualTo("eeeeeeeeeeee"); + assertThat(p.processName()).isEqualTo("train.py"); + assertThat(p.memoryUsedMiB()).isEqualTo(2); + assertThat(p.elapsedSeconds()).isZero(); + } + + @Test + @DisplayName("소유자를 찾으면 이름·계정·사용자·신청 번호를 채우고, 프로세스는 GPU 메모리 큰 순이다") + void resolvedOwnerAndSortedByMemory() { + Owner owner = new Owner("u7", 7L, "홍길동", "u7", 70L); + Sample small = s(1 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_UNKNOWN, "pid", "1", "process_name", "/a"); + Sample big = s(5 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_KNOWN, "pid", "11", "process_name", "/b/python"); + when(myGpuUsageService.snapshot("farm6")).thenReturn(snap(true, List.of(small, big), Map.of(C_KNOWN, owner))); + + List procs = service.getGpuUsage("farm6").get(0).gpus().get(0).processes(); + + assertThat(procs).hasSize(2); + AdminGpuUsageResponseDTO.Process first = procs.get(0); + assertThat(first.ownerName()).isEqualTo("홍길동"); + assertThat(first.ubuntuUsername()).isEqualTo("u7"); + assertThat(first.userId()).isEqualTo(7L); + assertThat(first.requestId()).isEqualTo(70L); + assertThat(first.processName()).isEqualTo("python"); + assertThat(first.elapsedSeconds()).isEqualTo(3600); + assertThat(procs.get(1).ownerName()).isNull(); + } + + @Test + @DisplayName("노드 목록이 비어 있으면 빈 목록을 준다") + void emptyNodeListReturnsEmpty() { + when(nodeRepository.findAll()).thenReturn(List.of()); + + assertThat(service.getGpuUsage(null)).isEmpty(); + assertThat(service.getGpuUsage(" ")).isEmpty(); + } + + @Test + @DisplayName("전체 조회는 DB의 모든 노드를 이름순으로 주고, 한 노드가 실패해도 그 노드만 빈 GPU로 표시한다") + void allNodesOneFailureIsolated() { + when(nodeRepository.findAll()).thenReturn(List.of(new Node("farm2", null, 1, 1), new Node("farm1", null, 1, 1))); + when(myGpuUsageService.snapshot("farm1")).thenThrow(new RuntimeException("prom down")); + when(myGpuUsageService.snapshot("farm2")).thenReturn(snap(true, List.of(), Map.of())); + + List all = service.getGpuUsage(null); + + assertThat(all).extracting(AdminGpuUsageResponseDTO::nodeName).containsExactly("farm1", "farm2"); + assertThat(all.get(0).processesAvailable()).isFalse(); + assertThat(all.get(0).gpus()).isEmpty(); + assertThat(all.get(1).gpus()).hasSize(1); + } + + @Test + @DisplayName("DB에 없는 노드를 지정하면 404") + void unknownNodeIsNotFound() { + when(nodeRepository.existsById("ghost")).thenReturn(false); + + assertThatThrownBy(() -> service.getGpuUsage("ghost")) + .isInstanceOf(BusinessException.class) + .hasFieldOrPropertyWithValue("errorCode", ErrorCode.RESOURCE_NOT_FOUND); + } + + @Test + @DisplayName("노드를 지정했는데 GPU 지표 조회가 실패하면 502(EXTERNAL_API_ERROR)") + void singleNodeMetricFailureIsBadGateway() { + when(nodeRepository.existsById(anyString())).thenReturn(true); + when(myGpuUsageService.snapshot("farm6")).thenThrow(new RuntimeException("prom down")); + + assertThatThrownBy(() -> service.getGpuUsage("farm6")) + .isInstanceOf(BusinessException.class) + .hasFieldOrPropertyWithValue("errorCode", ErrorCode.EXTERNAL_API_ERROR); + } +} diff --git a/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageServiceTest.java b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageServiceTest.java new file mode 100644 index 00000000..632fd84c --- /dev/null +++ b/src/test/java/DGU_AI_LAB/admin_be/domain/monitoring/service/MyGpuUsageServiceTest.java @@ -0,0 +1,287 @@ +package DGU_AI_LAB.admin_be.domain.monitoring.service; + +import DGU_AI_LAB.admin_be.domain.monitoring.client.PrometheusClient; +import DGU_AI_LAB.admin_be.domain.monitoring.client.PrometheusClient.Sample; +import DGU_AI_LAB.admin_be.domain.monitoring.dto.response.MyGpuUsageResponseDTO; +import DGU_AI_LAB.admin_be.domain.nodes.entity.Node; +import DGU_AI_LAB.admin_be.domain.nodes.repository.NodeRepository; +import DGU_AI_LAB.admin_be.domain.requests.entity.Request; +import DGU_AI_LAB.admin_be.domain.requests.entity.Status; +import DGU_AI_LAB.admin_be.domain.requests.repository.RequestRepository; +import DGU_AI_LAB.admin_be.domain.users.entity.User; +import DGU_AI_LAB.admin_be.error.exception.BusinessException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class MyGpuUsageServiceTest { + + @Mock PrometheusClient prometheus; + @Mock RequestRepository requestRepository; + @Mock NodeRepository nodeRepository; + @InjectMocks MyGpuUsageService service; + + private static final String C_MINE = "a".repeat(64); + private static final String C_BOB = "b".repeat(64); + private static final String C_AMY = "c".repeat(64); + private static final String C_DOCKER = "d".repeat(64); + private static final String C_UNKNOWN = "e".repeat(64); + + private Request request(Long userId, String pod, String node) { + User user = mock(User.class); + when(user.getUserId()).thenReturn(userId); + when(user.getUbuntuUsername()).thenReturn("u" + userId); + Request r = mock(Request.class); + when(r.getUser()).thenReturn(user); + when(r.getPodName()).thenReturn(pod); + when(r.getNodeName()).thenReturn(node); + when(r.getStatus()).thenReturn(Status.FULFILLED); + return r; + } + + @org.junit.jupiter.api.BeforeEach + void exporterUpByDefault() { + // 대부분의 테스트는 exporter가 이 노드에서 프로세스 목록을 읽고 있다고 가정한다. + when(prometheus.querySeries(contains("gpu_process_metrics_up"))).thenReturn(List.of( + s(1, "hostname", "FARM6"))); + } + + private static Sample s(double v, String... kv) { + java.util.Map m = new java.util.LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) m.put(kv[i], kv[i + 1]); + return new Sample(m, v); + } + + @Test + @DisplayName("내 프로세스는 전부, 다른 사용자는 사람 수와 메모리 합계만 담고 식별 정보는 응답에 없다") + void aggregatesOthersWithoutIdentity() throws Exception { + Request mine = request(1L, "pod-mine", "node-a"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(mine)); + when(prometheus.querySeries(contains("DCGM_FI_DEV_GPU_UTIL"))).thenReturn(List.of( + s(40, "UUID", "GPU-1", "gpu", "0", "modelName", "RTX 3090"), + s(5, "UUID", "GPU-2", "gpu", "1", "modelName", "RTX 3090"))); + when(prometheus.querySeries(contains("DCGM_FI_DEV_FB_USED"))).thenReturn(List.of( + s(10240, "UUID", "GPU-1"), s(0, "UUID", "GPU-2"))); + when(prometheus.querySeries(contains("DCGM_FI_DEV_FB_FREE"))).thenReturn(List.of( + s(14336, "UUID", "GPU-1"), s(24576, "UUID", "GPU-2"))); + when(prometheus.querySeries(contains("gpu_process_memory_used_bytes"))).thenReturn(List.of( + s(2 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_MINE, "pid", "11", "process_name", "/usr/bin/python"), + s(3 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_BOB, "pid", "22", "process_name", "/home/bob/train"), + s(4 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_BOB, "pid", "23", "process_name", "/home/bob/eval"), + s(1 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_AMY, "pid", "33", "process_name", "/home/amy/a"), + // 옛 Docker 컨테이너(사용자 dave)와 소유자를 못 찾는 컨테이너도 다른 사용자로 센다. + s(5 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_DOCKER, "pid", "44", "process_name", "/home/dave/x"), + s(6 * 1048576.0, "gpu_uuid", "GPU-1", "container_id", C_UNKNOWN, "pid", "55", "process_name", "/x"))); + when(prometheus.querySeries(contains("gpu_process_elapsed_seconds"))).thenReturn(List.of( + s(3600, "gpu_uuid", "GPU-1", "container_id", C_MINE, "pid", "11"))); + when(prometheus.querySeries(contains("kube_pod_container_info"))).thenReturn(List.of( + s(1, "container_id", "containerd://" + C_MINE, "pod", "pod-mine"), + s(1, "container_id", "containerd://" + C_BOB, "pod", "pod-secret-x"), + s(1, "container_id", "containerd://" + C_AMY, "pod", "pod-secret-y"))); + when(prometheus.querySeries(contains("docker_gpu_user_container_info"))).thenReturn(List.of( + s(1, "container_id", C_DOCKER.substring(0, 12), "gpu_uuid", "GPU-1", "ubuntu_username", "dave"))); + Request bob = request(2L, "pod-secret-x", "node-a"); + Request amy = request(3L, "pod-secret-y", "node-a"); + when(requestRepository.findAllByPodNameIn(anyCollection())).thenReturn(List.of(mine, bob, amy)); + + MyGpuUsageResponseDTO dto = service.getMyGpuUsage(10L, 1L); + + assertThat(dto.gpus()).hasSize(2); + MyGpuUsageResponseDTO.Gpu g0 = dto.gpus().get(0); + assertThat(g0.utilizationPercent()).isEqualTo(40); + assertThat(g0.memoryTotalMiB()).isEqualTo(24576); + assertThat(g0.myProcesses()).containsExactly(new MyGpuUsageResponseDTO.Process("python", 3600, 2)); + assertThat(g0.otherUserCount()).isEqualTo(4); + assertThat(g0.otherMemoryMiB()).isEqualTo(19); + assertThat(dto.gpus().get(1).otherUserCount()).isZero(); + + String json = new ObjectMapper().writeValueAsString(dto); + assertThat(json).doesNotContain("secret").doesNotContain("bob").doesNotContain("amy") + .doesNotContain("train").doesNotContain("dave").doesNotContain("\"pid\""); + } + + @Test + @DisplayName("남의 신청이면 거부한다") + void rejectsOthersRequest() { + Request other = mock(Request.class); + User u = mock(User.class); + when(u.getUserId()).thenReturn(2L); + when(other.getUser()).thenReturn(u); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(other)); + + assertThatThrownBy(() -> service.getMyGpuUsage(10L, 1L)).isInstanceOf(BusinessException.class); + } + + @Test + @DisplayName("실행 중(FULFILLED)이 아닌 신청은 pod·node가 남아 있어도 조회하지 못한다") + void rejectsNonFulfilled() { + Request deleted = request(1L, "pod-old", "node-a"); + when(deleted.getStatus()).thenReturn(Status.DELETED); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(deleted)); + + assertThatThrownBy(() -> service.getMyGpuUsage(10L, 1L)).isInstanceOf(BusinessException.class); + org.mockito.Mockito.verifyNoInteractions(prometheus); + } + + private void stubGpu(String uuid, String idx, String model, double util) { + when(prometheus.querySeries(contains("DCGM_FI_DEV_GPU_UTIL"))).thenReturn(List.of( + s(util, "UUID", uuid, "gpu", idx, "modelName", model))); + when(prometheus.querySeries(contains("DCGM_FI_DEV_FB_USED"))).thenReturn(List.of(s(1, "UUID", uuid))); + when(prometheus.querySeries(contains("DCGM_FI_DEV_FB_FREE"))).thenReturn(List.of(s(1, "UUID", uuid))); + } + + @Test + @DisplayName("프로세스 지표만 실패하면 GPU는 보이되 processesAvailable=false로 '모름'을 알린다") + void processFailureIsFlagged() { + Request r = request(1L, "p", "farm6"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + stubGpu("GPU-1", "0", "NVIDIA RTX A5000", 3); + when(prometheus.querySeries(contains("gpu_process_memory_used_bytes"))).thenThrow(new RuntimeException("down")); + + MyGpuUsageResponseDTO dto = service.getMyGpuUsage(10L, 1L); + + assertThat(dto.gpus()).hasSize(1); + assertThat(dto.processesAvailable()).isFalse(); + } + + @Test + @DisplayName("exporter의 gpu_process_metrics_up 시리즈가 없거나 0이면 processesAvailable=false, 있으면 true") + void processesAvailableFollowsExporterMarker() { + Request r = request(1L, "p", "farm6"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + stubGpu("GPU-1", "0", "m", 1); + when(prometheus.querySeries(contains("gpu_process_memory_used_bytes"))).thenReturn(List.of()); + + // 지표는 있는데 프로세스가 없을 뿐: 사용 가능 + assertThat(service.getMyGpuUsage(10L, 1L).processesAvailable()).isTrue(); + // 선택자는 실제 exporter 라벨(hostname=FARM6)을 대소문자 구분 없이 찾는다. + org.mockito.Mockito.verify(prometheus).querySeries( + "gpu_process_metrics_up{hostname=~\"(?i)farm6([.].*)?\"}"); + + // exporter가 없거나 옛 버전이라 시리즈 자체가 없음 + when(prometheus.querySeries(contains("gpu_process_metrics_up"))).thenReturn(List.of()); + assertThat(new MyGpuUsageService(prometheus, requestRepository, nodeRepository) + .getMyGpuUsage(10L, 1L).processesAvailable()).isFalse(); + + // 프로세스 목록 읽기 실패(0) + when(prometheus.querySeries(contains("gpu_process_metrics_up"))).thenReturn(List.of(s(0, "hostname", "FARM6"))); + assertThat(new MyGpuUsageService(prometheus, requestRepository, nodeRepository) + .getMyGpuUsage(10L, 1L).processesAvailable()).isFalse(); + } + + @Test + @DisplayName("노드에 등록된 모델의 GPU만 보인다(표기 차이는 무시), 노드 정보가 없으면 전부 보인다") + void filtersGpusByNodeModels() { + Request r = request(1L, "p", "farm6"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + when(prometheus.querySeries(contains("DCGM_FI_DEV_GPU_UTIL"))).thenReturn(List.of( + s(1, "UUID", "GPU-1", "gpu", "0", "modelName", "NVIDIA GeForce RTX 3090"), + s(1, "UUID", "GPU-2", "gpu", "1", "modelName", "NVIDIA RTX A5000"))); + Node node = mock(Node.class); + when(node.getGpuModels()).thenReturn(List.of("RTX 3090")); + when(nodeRepository.findByIdWithGpus("farm6")).thenReturn(Optional.of(node)); + + assertThat(service.getMyGpuUsage(10L, 1L).gpus()).extracting(MyGpuUsageResponseDTO.Gpu::index).containsExactly(0); + + when(nodeRepository.findByIdWithGpus("farm6")).thenReturn(Optional.empty()); + MyGpuUsageService fresh = new MyGpuUsageService(prometheus, requestRepository, nodeRepository); + assertThat(fresh.getMyGpuUsage(10L, 1L).gpus()).hasSize(2); + } + + @Test + @DisplayName("DCGM Hostname 선택자는 실제 라벨(farm6)에 대소문자·FQDN 차이를 허용하고 위험 문자는 이스케이프한다") + void hostSelectorFormat() { + assertThat(MyGpuUsageService.hostSelector("Hostname", "farm6")).isEqualTo("{Hostname=~\"(?i)farm6([.].*)?\"}"); + assertThat(MyGpuUsageService.hostSelector("Hostname", "a.b")).isEqualTo("{Hostname=~\"(?i)a[.]b([.].*)?\"}"); + assertThat(MyGpuUsageService.hostSelector("Hostname", "x\"y")).isEqualTo("{Hostname=\"x\\\"y\"}"); + assertThat(MyGpuUsageService.normalizeModel("NVIDIA GeForce RTX 3090")).isEqualTo("RTX 3090"); + assertThat(MyGpuUsageService.normalizeModel("RTX 3090 Ti")).isNotEqualTo("RTX 3090"); + } + + @Test + @DisplayName("컨테이너 ID가 없는 프로세스는 PID별로 세고, 같은 컨테이너 ID 없는 것끼리 한 명으로 뭉치지 않는다") + void missingContainerIdCountedByPid() { + Request r = request(1L, "p", "farm6"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + stubGpu("GPU-1", "0", "m", 1); + when(prometheus.querySeries(contains("gpu_process_memory_used_bytes"))).thenReturn(List.of( + s(1048576.0, "gpu_uuid", "GPU-1", "pid", "1"), + s(1048576.0, "gpu_uuid", "GPU-1", "pid", "2"), + s(1048576.0, "gpu_uuid", "GPU-1"))); + + MyGpuUsageResponseDTO.Gpu g = service.getMyGpuUsage(10L, 1L).gpus().get(0); + + assertThat(g.otherUserCount()).isEqualTo(2); + assertThat(g.otherMemoryMiB()).isEqualTo(3); + } + + @Test + @DisplayName("Docker 소유자는 같은 GPU 시계열일 때만 단축 ID로 맞춘다") + void dockerOwnerMustMatchGpu() { + Request r = request(1L, "p", "farm6"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + stubGpu("GPU-1", "0", "m", 1); + when(prometheus.querySeries(contains("gpu_process_memory_used_bytes"))).thenReturn(List.of( + s(1048576.0, "gpu_uuid", "GPU-1", "container_id", C_DOCKER, "pid", "9"))); + // 같은 단축 ID이지만 다른 GPU의 시계열이면 소유자로 보지 않는다. + when(prometheus.querySeries(contains("docker_gpu_user_container_info"))).thenReturn(List.of( + s(1, "container_id", C_DOCKER.substring(0, 12), "gpu_uuid", "GPU-OTHER", "ubuntu_username", "u1"))); + + MyGpuUsageResponseDTO.Gpu g = service.getMyGpuUsage(10L, 1L).gpus().get(0); + + assertThat(g.myProcesses()).isEmpty(); + assertThat(g.otherUserCount()).isEqualTo(1); + } + + @Test + @DisplayName("같은 노드를 짧은 시간에 다시 부르면 Prometheus를 다시 부르지 않는다") + void cachesPerNode() { + Request r = request(1L, "p", "farm6"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + stubGpu("GPU-1", "0", "m", 1); + + service.getMyGpuUsage(10L, 1L); + service.getMyGpuUsage(10L, 1L); + + org.mockito.Mockito.verify(prometheus, org.mockito.Mockito.times(1)).querySeries(contains("DCGM_FI_DEV_GPU_UTIL")); + } + + @Test + @DisplayName("Pod가 아직 없으면 빈 목록") + void noPodMeansEmpty() { + Request r = request(1L, null, null); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + + assertThat(service.getMyGpuUsage(10L, 1L).gpus()).isEmpty(); + } + + @Test + @DisplayName("GPU 지표 조회가 실패하면 오류") + void metricsFailureMeansEmpty() { + Request r = request(1L, "p", "node-a"); + when(requestRepository.findByIdWithUser(10L)).thenReturn(Optional.of(r)); + when(prometheus.querySeries(contains("DCGM_FI_DEV_GPU_UTIL"))).thenThrow(new RuntimeException("down")); + + // 빈 목록('GPU 없음')으로 숨기지 않고 오류로 알린다. + assertThatThrownBy(() -> service.getMyGpuUsage(10L, 1L)).isInstanceOf(BusinessException.class); + } +} diff --git a/src/test/java/DGU_AI_LAB/admin_be/global/auth/AdminPathAccessDeniedTest.java b/src/test/java/DGU_AI_LAB/admin_be/global/auth/AdminPathAccessDeniedTest.java index f53cfea4..33dc340c 100644 --- a/src/test/java/DGU_AI_LAB/admin_be/global/auth/AdminPathAccessDeniedTest.java +++ b/src/test/java/DGU_AI_LAB/admin_be/global/auth/AdminPathAccessDeniedTest.java @@ -86,6 +86,27 @@ void imageRegistration_withUserRole_isRejected() { assertThat(legacy.getStatusCode()).isEqualTo(HttpStatus.METHOD_NOT_ALLOWED); } + @Test + @DisplayName("일반 사용자 토큰으로 관리자 GPU 사용 현황을 부르면 403이다") + void adminGpuUsage_withUserRole_returnsForbidden() { + User user = userRepository.save(User.builder() + .email("gpu-user@dgu.ac.kr") + .password("encoded") + .name("일반사용자") + .studentId("2021001235") + .phone("010-1111-3333") + .department("컴퓨터공학과") + .build()); + HttpHeaders headers = new HttpHeaders(); + headers.setBearerAuth(jwtProvider.getIssueToken(user.getUserId(), true)); + + ResponseEntity response = restTemplate.exchange( + "/api/admin/gpu-usage", HttpMethod.GET, new HttpEntity<>(headers), String.class); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.FORBIDDEN); + assertThat(response.getBody()).contains("ACCESS_DENIED"); + } + @Test @DisplayName("토큰 없이 관리자 API를 부르면 401을 받는다") void adminPath_withoutToken_returnsUnauthorized() {