diff --git a/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusDetectionService.java b/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusDetectionService.java index f8687ef..636f412 100644 --- a/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusDetectionService.java +++ b/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusDetectionService.java @@ -1,18 +1,90 @@ package com.eactive.eai.rms.ext.djb.apistatus; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import com.eactive.apim.portal.common.entity.Auditable; +import com.eactive.apim.portal.djb.apistatus.incident.entity.DjbApistatusIncident; +import com.eactive.apim.portal.djb.apistatus.incident.entity.DjbApistatusIncidentApi; +import com.eactive.apim.portal.djb.apistatus.incident.entity.DjbApistatusIncidentTimeline; +import com.eactive.apim.portal.djb.apistatus.incident.entity.IncidentKind; +import com.eactive.apim.portal.djb.apistatus.incident.entity.IncidentState; +import com.eactive.apim.portal.djb.apistatus.incident.repository.DjbApistatusIncidentApiRepository; +import com.eactive.apim.portal.djb.apistatus.incident.repository.DjbApistatusIncidentRepository; +import com.eactive.apim.portal.djb.apistatus.incident.repository.DjbApistatusIncidentTimelineRepository; +import com.eactive.apim.portal.portalNotice.entity.PortalNotice; +import com.eactive.eai.rms.data.entity.onl.apim.portalnotice.PortalNoticeService; +import com.eactive.eai.rms.data.entity.onl.djb.apistatus.ApiNameRow; + +/** + * API 상태 탐지 결과를 개발자포탈 API Status 화면용 장애 데이터로 반영한다. + * + *

탐지 자체는 {@link ApiStatusService} 가 수행하며, 본 서비스는 그 결과를 + * {@code DJB_APISTATUS_INCIDENT} / {@code _API} / {@code _TIMELINE} + {@code PTL_NOTICE} 초안으로 기록한다.

+ * + *

점검(CONTROL_*) 은 관리자가 공지사항으로 등록한 점검 일정이 원천이다 + * ({@code ApiStatusRepository.findApiStatusEvents} 가 INCIDENT 기간을 읽어 CTRL_YN 을 판정). + * 따라서 CONTROL 이벤트를 다시 기록하면 순환이 되므로 기록하지 않는다.

+ */ @Service -@Transactional +@Transactional(transactionManager = "transactionManagerForEMS", rollbackFor = Exception.class) public class ApiStatusDetectionService { private static final Logger log = LoggerFactory.getLogger(ApiStatusDetectionService.class); - + + public static final String EVENT_CONTROL_START = "CONTROL_START"; + public static final String EVENT_CONTROL_END = "CONTROL_END"; + public static final String EVENT_ERROR_START = "ERROR_START"; + public static final String EVENT_ERROR_END = "ERROR_END"; + public static final String EVENT_DELAY_START = "DELAY_START"; + public static final String EVENT_DELAY_END = "DELAY_END"; + + /** INTERFACE_ID 멱등 윈도우 (분) */ + private static final int IDEMPOTENT_WINDOW_MINUTES = 5; + + /** 종결 상태 - 신규 탐지 병합 대상에서 제외 */ + private static final List CLOSED_STATES = + Collections.unmodifiableList(Arrays.asList(IncidentState.RESOLVED, IncidentState.CANCELED)); + + private static final String AUTHOR_SYSTEM = "SYSTEM"; + private static final String DETECTED_BY_AUTO = "AUTO"; + private static final String NOTICE_TYPE_INCIDENT = "3"; + private static final int NOTICE_SUBJECT_MAX_LENGTH = 255; + + @Autowired + private DjbApistatusIncidentRepository incidentRepository; + + @Autowired + private DjbApistatusIncidentApiRepository incidentApiRepository; + + @Autowired + private DjbApistatusIncidentTimelineRepository incidentTimelineRepository; + + @Autowired + private PortalNoticeService portalNoticeService; + + @Autowired + private ApiStatusRepository apiStatusRepository; + /** * 이벤트 감지 * @param event "CONTROL_START" : 점검시작 @@ -25,17 +97,313 @@ public class ApiStatusDetectionService { */ public void detect(String event, List apiIds) { log.debug("이벤트 감지: {} {}", event, apiIds); - } - - // API FSM 내 아직 장애로 남아있는 API 목록 - public void remainDownApiIds() { - - } - - - // 정상 동작 - detect()에서 일괄 처리 (복구 = 점검종료, 장애종료, 지연종료) -// public void recovered(String event, String[] apiIds) { -// -// } -} \ No newline at end of file + if (StringUtils.isBlank(event) || apiIds == null || apiIds.isEmpty()) { + return; + } + + List targets = apiIds.stream() + .filter(StringUtils::isNotBlank) + .distinct() + .sorted() + .collect(Collectors.toList()); + if (targets.isEmpty()) { + return; + } + + switch (event) { + case EVENT_ERROR_START: + openIncident(event, targets, "에러율 임계 초과", "장애"); + break; + case EVENT_DELAY_START: + openIncident(event, targets, "응답시간 임계 초과", "응답지연"); + break; + case EVENT_ERROR_END: + case EVENT_DELAY_END: + markRecovered(event, targets); + break; + case EVENT_CONTROL_START: + case EVENT_CONTROL_END: + // 점검은 관리자 등록 공지가 원천 - 재기록하지 않음 + log.debug("점검 이벤트는 장애 데이터로 기록하지 않음: {} {}", event, targets); + break; + default: + log.warn("알 수 없는 이벤트: {}", event); + } + } + + /** + * 미종결 장애로 남아있는 API 목록 (탐지 Job 폴링용) + */ + @Transactional(transactionManager = "transactionManagerForEMS", readOnly = true) + public List remainDownApiIds() { + List openIncidents = + incidentRepository.findByKindAndStateNotInOrderByStartedAtDesc(IncidentKind.INCIDENT, CLOSED_STATES); + if (openIncidents.isEmpty()) { + return Collections.emptyList(); + } + + List incidentIds = openIncidents.stream() + .map(DjbApistatusIncident::getIncidentId) + .collect(Collectors.toList()); + + return incidentApiRepository.findByIncidentIdInOrderByIncidentIdAscApiIdAsc(incidentIds).stream() + .filter(api -> api.getRecoveredAt() == null) + .map(DjbApistatusIncidentApi::getApiId) + .distinct() + .sorted() + .collect(Collectors.toList()); + } + + // ────────────── 장애 발생 ────────────── + + private void openIncident(String event, List apiIds, String summary, String titleKeyword) { + LocalDateTime now = LocalDateTime.now(); + String interfaceId = makeInterfaceId(event, apiIds, now); + + // 1) 동일 윈도우 재탐지 - 타임라인만 남기고 종료 + Optional duplicated = incidentRepository.findByInterfaceId(interfaceId); + if (duplicated.isPresent()) { + appendTimeline(duplicated.get().getIncidentId(), null, + "재탐지 신호 수신 (" + event + ")", now); + return; + } + + // 2) 이미 열려있는 자동 등록 장애가 해당 API 를 포함하면 영향 API 만 병합 + // (관리자가 직접 작성한 장애 공지는 건드리지 않는다) + List openIncidents = incidentRepository + .findOpenByApiIds(IncidentKind.INCIDENT, DETECTED_BY_AUTO, CLOSED_STATES, apiIds); + if (!openIncidents.isEmpty()) { + mergeIntoOpenIncident(openIncidents.get(0), event, apiIds, now); + return; + } + + // 3) 신규 장애 + 공지 초안 생성 + Map apiNames = resolveApiNames(apiIds); + String title = buildTitle(titleKeyword, apiIds, apiNames); + + PortalNotice notice = createDraftNotice(title, summary, apiIds, apiNames, now); + + DjbApistatusIncident incident = new DjbApistatusIncident(); + incident.setKind(IncidentKind.INCIDENT); + incident.setState(IncidentState.INVESTIGATING); + incident.setTitle(title); + incident.setSummary(summary); + incident.setStartedAt(now); + incident.setDetectedBy(DETECTED_BY_AUTO); + incident.setInterfaceId(interfaceId); + incident.setNoticeId(notice.getId()); + incident.setDraftYn("Y"); + incident.setFixYn("N"); + stampAudit(incident, now); + + DjbApistatusIncident saved; + try { + saved = incidentRepository.saveAndFlush(incident); + } catch (DataIntegrityViolationException e) { + // INTERFACE_ID UNIQUE 충돌 - 동시 탐지. 공지 초안 정리 후 타임라인만 남긴다 + log.info("장애 중복 감지 (interfaceId={}) - 기존 장애에 타임라인 추가", interfaceId, e); + portalNoticeService.deleteById(notice.getId()); + incidentRepository.findByInterfaceId(interfaceId).ifPresent(existing -> + appendTimeline(existing.getIncidentId(), null, + "재탐지 신호 수신 (" + event + ")", now)); + return; + } + + saveIncidentApis(saved.getIncidentId(), apiIds, apiNames, now); + appendTimeline(saved.getIncidentId(), IncidentState.INVESTIGATING, + summary + " 자동 감지 (" + event + ")\n영향 API: " + String.join(", ", apiIds), now); + + log.info("자동 장애 등록: incidentId={}, noticeId={}, apis={}", + saved.getIncidentId(), notice.getId(), apiIds); + } + + private void mergeIntoOpenIncident(DjbApistatusIncident incident, String event, + List apiIds, LocalDateTime now) { + Long incidentId = incident.getIncidentId(); + + Set mapped = incidentApiRepository.findByIncidentIdOrderByApiId(incidentId).stream() + .map(DjbApistatusIncidentApi::getApiId) + .collect(Collectors.toCollection(HashSet::new)); + + List added = apiIds.stream() + .filter(apiId -> !mapped.contains(apiId)) + .collect(Collectors.toList()); + + if (added.isEmpty()) { + appendTimeline(incidentId, null, "재탐지 신호 수신 (" + event + ")", now); + log.debug("기존 장애에 이미 포함된 API - 타임라인만 추가: incidentId={}, apis={}", incidentId, apiIds); + return; + } + + saveIncidentApis(incidentId, added, resolveApiNames(added), now); + appendTimeline(incidentId, null, + "영향 API 추가 감지 (" + event + ")\n" + String.join(", ", added), now); + log.info("기존 장애에 영향 API 병합: incidentId={}, addedApis={}", incidentId, added); + } + + // ────────────── 복구 ────────────── + + private void markRecovered(String event, List apiIds) { + LocalDateTime now = LocalDateTime.now(); + + List candidates = + incidentApiRepository.findByApiIdInAndRecoveredAtIsNull(apiIds); + if (candidates.isEmpty()) { + log.debug("복구 대상 장애 없음: {} {}", event, apiIds); + return; + } + + Map openIncidents = incidentRepository + .findByKindAndStateNotInOrderByStartedAtDesc(IncidentKind.INCIDENT, CLOSED_STATES).stream() + .collect(Collectors.toMap(DjbApistatusIncident::getIncidentId, incident -> incident)); + + Map> recoveredByIncident = new HashMap<>(); + for (DjbApistatusIncidentApi api : candidates) { + if (!openIncidents.containsKey(api.getIncidentId())) { + continue; + } + api.setRecoveredAt(now); + api.setLastModifiedBy(AUTHOR_SYSTEM); + api.setLastModifiedDate(now); + incidentApiRepository.save(api); + + recoveredByIncident.computeIfAbsent(api.getIncidentId(), key -> new ArrayList<>()) + .add(api.getApiId()); + } + + if (recoveredByIncident.isEmpty()) { + log.debug("복구 대상 장애 없음 (미종결 장애 미포함): {} {}", event, apiIds); + return; + } + + for (Map.Entry> entry : recoveredByIncident.entrySet()) { + Long incidentId = entry.getKey(); + List recoveredApis = entry.getValue(); + DjbApistatusIncident incident = openIncidents.get(incidentId); + + boolean allRecovered = incidentApiRepository.countByIncidentIdAndRecoveredAtIsNull(incidentId) == 0; + IncidentState nextState = allRecovered ? IncidentState.RESOLVED : IncidentState.MONITORING; + + if (incident.getState() != nextState) { + incident.setPreviousState(incident.getState()); + incident.setState(nextState); + } + if (allRecovered) { + incident.setEndAt(now); + } + incident.setLastModifiedBy(AUTHOR_SYSTEM); + incident.setLastModifiedDate(now); + incidentRepository.save(incident); + + String body = (allRecovered ? "전체 복구 확인 (" : "일부 복구 확인 (") + event + ")\n" + + String.join(", ", recoveredApis); + appendTimeline(incidentId, nextState, body, now); + + log.info("자동 복구 처리: incidentId={}, state={}, apis={}", incidentId, nextState, recoveredApis); + } + } + + // ────────────── 내부 헬퍼 ────────────── + + private String makeInterfaceId(String event, List sortedApiIds, LocalDateTime now) { + long windowSeconds = IDEMPOTENT_WINDOW_MINUTES * 60L; + long flooredEpoch = now.atZone(ZoneId.systemDefault()).toEpochSecond() + / windowSeconds * windowSeconds; + String joined = String.join(",", sortedApiIds); + String interfaceId = event + ":" + joined + ":" + flooredEpoch; + + // INTERFACE_ID 는 VARCHAR2(200) - 초과 시 API 목록을 해시로 축약 + if (interfaceId.length() > 200) { + interfaceId = event + ":#" + Integer.toHexString(joined.hashCode()) + + ":" + sortedApiIds.size() + ":" + flooredEpoch; + } + return interfaceId; + } + + private Map resolveApiNames(List apiIds) { + Map names = new HashMap<>(); + try { + for (ApiNameRow row : apiStatusRepository.findApiNames(apiIds)) { + names.put(row.getEaisvcname(), row.getEaisvcdesc()); + } + } catch (Exception e) { + log.warn("API 명 조회 실패 - API ID 를 그대로 사용", e); + } + return names; + } + + private String apiLabel(String apiId, Map apiNames) { + return StringUtils.defaultIfBlank(apiNames.get(apiId), apiId); + } + + private String buildTitle(String titleKeyword, List apiIds, Map apiNames) { + String first = apiLabel(apiIds.get(0), apiNames); + if (apiIds.size() > 1) { + return "[자동감지] " + first + " 외 " + (apiIds.size() - 1) + "종 API " + titleKeyword; + } + return "[자동감지] " + first + " API " + titleKeyword; + } + + private PortalNotice createDraftNotice(String title, String summary, List apiIds, + Map apiNames, LocalDateTime now) { + StringBuilder detail = new StringBuilder(); + detail.append("

").append(summary).append(" 로 자동 감지된 장애입니다. 관리자 검수 후 정식 게시됩니다.

"); + detail.append("

영향 API

    "); + for (String apiId : apiIds) { + detail.append("
  • ").append(apiLabel(apiId, apiNames)).append(" (").append(apiId).append(")
    • "); + } + detail.append("
"); + + PortalNotice notice = new PortalNotice(); + notice.setId(null); + notice.setNoticeSubject(StringUtils.abbreviate(title, NOTICE_SUBJECT_MAX_LENGTH)); + notice.setNoticeDetail(detail.toString()); + notice.setNoticeType(NOTICE_TYPE_INCIDENT); + notice.setUseYn("N"); + notice.setFixYn("N"); + notice.setReadCount(0L); + notice.setInquirerName(AUTHOR_SYSTEM); + notice.setCreatedBy(AUTHOR_SYSTEM); + notice.setCreatedDate(now); + + portalNoticeService.save(notice); + return notice; + } + + private void saveIncidentApis(Long incidentId, Collection apiIds, + Map apiNames, LocalDateTime now) { + List rows = new ArrayList<>(); + for (String apiId : apiIds) { + DjbApistatusIncidentApi api = new DjbApistatusIncidentApi(); + api.setIncidentId(incidentId); + api.setApiId(apiId); + api.setApiName(apiNames.get(apiId)); + stampAudit(api, now); + rows.add(api); + } + if (!rows.isEmpty()) { + incidentApiRepository.saveAll(rows); + } + } + + private void appendTimeline(Long incidentId, IncidentState stateAfter, String body, LocalDateTime now) { + DjbApistatusIncidentTimeline timeline = new DjbApistatusIncidentTimeline(); + timeline.setIncidentId(incidentId); + timeline.setEventAt(now); + timeline.setStateAfter(stateAfter); + timeline.setBody(StringUtils.abbreviate(body, 4000)); + timeline.setAuthorType(AUTHOR_SYSTEM); + timeline.setVisibleYn("Y"); + stampAudit(timeline, now); + incidentTimelineRepository.save(timeline); + } + + /** + * 스케줄러 스레드는 로그인 세션이 없어 AuditorAware 가 비어있다. CREATED_BY 는 NOT NULL 이므로 직접 채운다. + */ + private void stampAudit(Auditable entity, LocalDateTime now) { + entity.setCreatedBy(AUTHOR_SYSTEM); + entity.setCreatedDate(now); + } +} diff --git a/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusRepository.java b/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusRepository.java index 9a0f4bf..b0ed6c4 100644 --- a/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusRepository.java +++ b/src/main/java/com/eactive/eai/rms/ext/djb/apistatus/ApiStatusRepository.java @@ -4,7 +4,10 @@ import java.util.List; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import java.util.Collection; + import com.eactive.eai.data.jpa.BaseRepository; +import com.eactive.eai.rms.data.entity.onl.djb.apistatus.ApiNameRow; import com.eactive.eai.rms.data.entity.onl.djb.apistatus.ApiStatus; import com.eactive.eai.rms.data.entity.onl.djb.apistatus.ApiStatusEvent; @@ -73,4 +76,11 @@ public interface ApiStatusRepository extends BaseRepository { @Param("delayRangeMinute") int delayRangeMinute, @Param("delayAvgRespTime") int delayAvgRespTime ); + + /** + * EAI 서비스명 조회. 장애/점검 영향 API 의 API_NAME 캐시용. + */ + @Query(nativeQuery = true, value = + " SELECT EAISVCNAME, EAISVCDESC FROM TSEAIHE01 WHERE EAISVCNAME IN (:apiIds)") + List findApiNames(@Param("apiIds") Collection apiIds); } \ No newline at end of file diff --git a/src/main/java/com/eactive/eai/rms/onl/apim/messagerequest/MessageRequestManService.java b/src/main/java/com/eactive/eai/rms/onl/apim/messagerequest/MessageRequestManService.java index e78fab6..6629c40 100644 --- a/src/main/java/com/eactive/eai/rms/onl/apim/messagerequest/MessageRequestManService.java +++ b/src/main/java/com/eactive/eai/rms/onl/apim/messagerequest/MessageRequestManService.java @@ -82,9 +82,9 @@ public class MessageRequestManService extends BaseService { return t == null ? "" : t.format(f); } - /** 상태 검색 콤보 (PENDING/SENT/FAILED). */ + /** 상태 검색 콤보 (PENDING/SENT/FAILED/SKIPPED). SKIPPED 는 행번 아닌 ID 로 발송을 건너뛴 건. */ public List statusCombo() { - return Arrays.stream(new String[]{"PENDING", "SENT", "FAILED"}) + return Arrays.stream(new String[]{"PENDING", "SENT", "FAILED", "SKIPPED"}) .map(s -> { ComboVo v = new ComboVo(); v.setCode(s);