From d5289173c7619b05cdf246dac4083c8feadc9da9 Mon Sep 17 00:00:00 2001 From: happybell80 Date: Sun, 14 Sep 2025 14:00:42 +0900 Subject: [PATCH 1/2] docs: Add OpenSearch installation guide for centralized logging - Compare logging solutions (OpenSearch vs Loki vs ClickHouse) - Document Docker Compose configuration - Include Fluentd setup for log collection - Add system resource requirements --- .../250914_admin_opensearch_installation.md | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 troubleshooting/250914_admin_opensearch_installation.md diff --git a/troubleshooting/250914_admin_opensearch_installation.md b/troubleshooting/250914_admin_opensearch_installation.md new file mode 100644 index 0000000..cf55d99 --- /dev/null +++ b/troubleshooting/250914_admin_opensearch_installation.md @@ -0,0 +1,135 @@ +# OpenSearch 로그 시스템 구축 + +## 작성일: 2025-09-14 +## 작성자: admin +## 환경: Ubuntu 22.04, 51123 서버, Docker Compose + +## 요구사항 분석 + +### 목적 +- 51123/51124 서버 로그 중앙집중식 관리 +- 실시간 로그 검색 및 분석 +- 시각화 대시보드 제공 + +### 솔루션 비교 +| 항목 | OpenSearch | Loki+Grafana | Vector+ClickHouse | +|------|------------|--------------|-------------------| +| 메모리 | 2-4GB | 512MB-1GB | 1-2GB | +| 검색성능 | 풀텍스트 최강 | 라벨 기반 | SQL 지원 | +| Docker 통합 | Fluentd 필요 | Promtail 자동 | Vector 자동 | +| 대시보드 | OpenSearch Dashboards | Grafana | Grafana 연동 | + +**선택**: OpenSearch (복잡한 텍스트 검색 요구) + +## 시스템 확인 + +### 리소스 현황 +```bash +# 메모리: 29GB 총량, 23GB 가용 +free -h +# HDD: 916GB 총량, 869GB 가용 +df -h /mnt/hdd +``` + +## Docker Compose 설정 + +```yaml +# /home/admin/opensearch/docker-compose.yml +version: '3' +services: + opensearch: + image: opensearchproject/opensearch:2.11.0 + container_name: opensearch + environment: + - cluster.name=robeing-logs + - node.name=opensearch-node1 + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms2g -Xmx2g" + - DISABLE_SECURITY_PLUGIN=true # 개발환경 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - /mnt/hdd/opensearch/data:/usr/share/opensearch/data + ports: + - 9200:9200 + - 9600:9600 + networks: + - opensearch-net + + opensearch-dashboards: + image: opensearchproject/opensearch-dashboards:2.11.0 + container_name: opensearch-dashboards + ports: + - 5601:5601 + environment: + - OPENSEARCH_HOSTS=http://opensearch:9200 + - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true + networks: + - opensearch-net + + fluentd: + build: ./fluentd + container_name: fluentd + volumes: + - /mnt/hdd/logs:/var/log/system:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - ./fluentd/fluent.conf:/fluentd/etc/fluent.conf + environment: + - OPENSEARCH_HOST=opensearch + - OPENSEARCH_PORT=9200 + networks: + - opensearch-net + depends_on: + - opensearch + +networks: + opensearch-net: +``` + +## Fluentd 설정 + +```ruby +# /home/admin/opensearch/fluentd/fluent.conf + + @type tail + path /var/log/system/**/*.log + pos_file /fluentd/log/system.pos + tag system.logs + + @type multiline + format_firstline /^\d{4}-\d{2}-\d{2}/ + format1 /^(? + + + + @type opensearch + host "#{ENV['OPENSEARCH_HOST']}" + port "#{ENV['OPENSEARCH_PORT']}" + logstash_format true + logstash_prefix robeing + +``` + +## 실행 및 검증 + +```bash +# 디렉토리 생성 +mkdir -p /mnt/hdd/opensearch/data +chmod 777 /mnt/hdd/opensearch/data + +# 실행 +cd /home/admin/opensearch +docker compose up -d + +# 상태 확인 +curl -X GET "localhost:9200/_cluster/health?pretty" +``` + +## 다음 단계 +1. 인덱스 라이프사이클 정책 설정 (30일 보관) +2. 기존 로그 마이그레이션 스크립트 작성 +3. 알림 규칙 설정 (에러 임계값) \ No newline at end of file From 3978b19a4c9f24095d837f1d138f563ebbcb943e Mon Sep 17 00:00:00 2001 From: happybell80 Date: Sun, 14 Sep 2025 14:07:29 +0900 Subject: [PATCH 2/2] docs: Add server log status to OpenSearch installation guide --- plans/250911_main_db3_migration_plan.md | 10 +++++++++- .../250914_admin_opensearch_installation.md | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/plans/250911_main_db3_migration_plan.md b/plans/250911_main_db3_migration_plan.md index c60187e..e372782 100644 --- a/plans/250911_main_db3_migration_plan.md +++ b/plans/250911_main_db3_migration_plan.md @@ -106,4 +106,12 @@ FROM main_db3.conversation_logs WHERE user_id IS NOT NULL; - **team_id 필수**: user 테이블 INSERT 시 NOT NULL - **UUID 타입**: 모든 ID는 UUID 타입 유지 - **slack_user_id**: workspace_member로 이동 예정 -- **실행 시간**: 새벽 시간대 권장 (서비스 영향 최소화) \ No newline at end of file +- **실행 시간**: 새벽 시간대 권장 (서비스 영향 최소화) + +--- + +## 6. 실행 결과 (2025-09-11) + +### User 테이블 마이그레이션 완료 +- main_db3에서 12명 이관 (총 14명 중 2명 중복 제외) +- UUID 불일치: goeun2dc@gmail.com, info@company-x.partners (기존 main_db UUID 유지) \ No newline at end of file diff --git a/troubleshooting/250914_admin_opensearch_installation.md b/troubleshooting/250914_admin_opensearch_installation.md index cf55d99..9212916 100644 --- a/troubleshooting/250914_admin_opensearch_installation.md +++ b/troubleshooting/250914_admin_opensearch_installation.md @@ -29,8 +29,15 @@ free -h # HDD: 916GB 총량, 869GB 가용 df -h /mnt/hdd +# SSD: 228GB 총량, 158GB 가용 (27% 사용) +df -h / ``` +### 로그 현황 +- `/mnt/hdd/logs/`: 51124-server/, act_runner/, backup/, cleanup/, company-x/, nginx/ +- 종료된 컨테이너: skill-slack (10일 전), rb10508_test (6주 전) +- 자동 정리: 매일 4시, 30일 보관 + ## Docker Compose 설정 ```yaml