Skip to content

U
ubains-module-test
提交 5b9004b8 authored 作者: 陈泽健's avatar 陈泽健
浏览文件
操作

Merge branch 'develop' of http://git.ubainsyun.com/bing/ubains-module-test into develop

上级 122bc765 0396ae7f
展开全部 隐藏空白字符变更
内嵌 并排
正在显示 包含 3725 行增加 和 107 行删除
.claude/skills/ARM-CLUSTER-MONITOR/code/monitor_slave1_192.168.9.90.json 0 → 100644
浏览文件 @ 5b9004b8
{
"ip": "192.168.9.90",
"role": "slave1",
"status": "warning",
"cpu": "7.5%",
"memory": "89.6%",
"disk": "/data 33% (25G/79G), / 33% (2.6G/7.8G)",
"load": "1.72, 4.96, 4.77",
"uptime": "5 days, 17:49",
"containers": {
"total": 9,
"running": 9,
"abnormal": [],
"details": "utracker(Up 4d), redis-slave-1(Up 5d), redis-sentinel-2(Up 5d), unacos(Up 17h), uemqx(Up 5d), utengine(Up 5d), paperless(Up 5d), ujava2(Up 5d), umysql(Up 5d)"
},
"middleware": {
"mysql": "ok",
"redis": "ok",
"emqx": "ok",
"nacos": "ok"
},
"java_services": 9,
"log_errors": 0,
"alerts": [
"内存使用率89.6%(含缓存)，可用内存仅1661MB/15948MB(10.4%)，接近85%告警阈值"
],
"summary": "slave1节点整体运行正常，9个容器全部Running，4项中间件(MySQL/Redis/EMQX/Nacos)功能验证通过，CPU负载低(7.5%)，日志无ERROR。但内存使用率89.6%已达WARNING级别(>85%)，可用内存仅1.6GB，需关注内存消耗趋势。unacos容器17小时前重启过，建议排查重启原因。"
}
\ No newline at end of file
.claude/skills/CreateCMD/_run_claude.bat
浏览文件 @ 5b9004b8
@echo off
cd /d E:\github\ubains-module-test\develop
E:\nodejs\claude.cmd --permission-mode bypassPermissions
cd /d C:\PycharmData\ubains-module-test\develop
C:\nvm4w\nodejs\claude.cmd --permission-mode bypassPermissions
.claude/skills/X86-CLUSTER-MONITOR/code/arm_monitor_node3.py 0 → 100644
浏览文件 @ 5b9004b8
import paramiko
import json
import re
import time
HOST = '192.168.9.91'
PORT = 22
USER = 'openkylin'
PASS = 'Ubains@123'
def ssh_exec(ssh, cmd, timeout=30):
try:
stdin, stdout, stderr = ssh.exec_command(cmd, timeout=timeout)
out = stdout.read().decode('utf-8', errors='replace').strip()
err = stderr.read().decode('utf-8', errors='replace').strip()
return out, err
except Exception as e:
return '', str(e)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(HOST, port=PORT, username=USER, password=PASS, timeout=15)
result = {
'ip': HOST,
'role': 'slave2',
'status': 'normal',
'cpu': '',
'memory': '',
'disk': '',
'load': '',
'uptime': '',
'containers': {'total': 0, 'running': 0, 'abnormal': [], 'details': ''},
'middleware': {'mysql': 'skip', 'redis': 'skip', 'emqx': 'skip', 'nacos': 'skip'},
'java_services': 0,
'log_errors': 0,
'alerts': [],
'summary': ''
}
# Use C locale to avoid Chinese encoding issues
LC = 'export LC_ALL=C LANG=C; '
# 1. CPU
out, err = ssh_exec(ssh, LC + 'top -bn1|grep Cpu')
print('CPU raw:', repr(out))
cpu_usage = 0
if out:
m = re.search(r'(\d+\.\d+)\s+id', out)
if m:
cpu_usage = round(100 - float(m.group(1)), 1)
result['cpu'] = str(cpu_usage) + '%'
# 2. Memory - use C locale
out, err = ssh_exec(ssh, LC + 'free -m')
print('MEM raw:', repr(out))
mem_usage = 0
mem_total_mb = 0
mem_used_mb = 0
mem_avail_mb = 0
if out:
lines = out.split('\n')
for line in lines:
if line.startswith('Mem:'):
parts = line.split()
mem_total_mb = int(parts[1])
mem_used_mb = int(parts[2])
# available is the last column in modern free
mem_avail_mb = int(parts[6]) if len(parts) > 6 else int(parts[3])
mem_usage = round((1 - mem_avail_mb / mem_total_mb) * 100, 1)
break
print(f'MEM total:{mem_total_mb} used:{mem_used_mb} avail:{mem_avail_mb} usage:{mem_usage}%')
result['memory'] = str(mem_usage) + '%'
# 3. Disk
out, err = ssh_exec(ssh, LC + 'df -h')
print('DISK raw:', repr(out))
disk_info = []
disk_alerts = []
if out:
lines = out.split('\n')
for line in lines[1:]:
parts = line.split()
if len(parts) >= 6 and parts[0].startswith('/dev/'):
mount = parts[5]
usage = parts[4]
size = parts[1]
disk_info.append(f'{mount}:{usage}({size})')
usage_pct = usage.replace('%', '')
try:
if int(usage_pct) > 85:
disk_alerts.append(f'Disk {mount} usage {usage_pct}% > 85%')
except:
pass
result['disk'] = '; '.join(disk_info) if disk_info else 'N/A'
# 4. Load and Uptime
out, err = ssh_exec(ssh, LC + 'uptime')
print('UPTIME raw:', repr(out))
if out:
m = re.search(r'load average:\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)', out)
if m:
result['load'] = f'{m.group(1)},{m.group(2)},{m.group(3)}'
m2 = re.search(r'up\s+(.+?),\s*\d+\s+user', out)
if m2:
result['uptime'] = m2.group(1).strip()
# 5. Containers
out, err = ssh_exec(ssh, 'sudo docker ps -a --format "{{.Names}}\t{{.Status}}"', timeout=30)
print('DOCKER PS raw:', repr(out))
containers = []
running = 0
abnormal = []
if out:
for line in out.split('\n'):
if not line.strip():
continue
parts = line.split('\t')
if len(parts) >= 2:
name = parts[0]
status = parts[1]
containers.append(f'{name}({status})')
if 'Up' in status:
running += 1
else:
abnormal.append(name)
out2, err2 = ssh_exec(ssh, 'sudo docker stats --no-stream', timeout=30)
print('DOCKER STATS raw:', repr(out2))
# Check high CPU containers
high_cpu_containers = []
if out2:
for line in out2.split('\n')[1:]:
if not line.strip():
continue
parts = line.split()
if len(parts) >= 3:
name = parts[1]
cpu_str = parts[2]
try:
cpu_val = float(cpu_str.replace('%', ''))
if cpu_val > 100:
high_cpu_containers.append(f'{name}:{cpu_str}')
except:
pass
total = len(containers)
result['containers']['total'] = total
result['containers']['running'] = running
result['containers']['abnormal'] = abnormal
result['containers']['details'] = f'Total:{total}, Running:{running}, Stopped:{total-running}'
if high_cpu_containers:
result['containers']['details'] += f', HighCPU:{",".join(high_cpu_containers)}'
# 6. Middleware
# MySQL
out, err = ssh_exec(ssh, "sudo docker exec umysql mysql -uroot -p'dNrprU&2S' -e 'SELECT 1;'", timeout=15)
print('MySQL raw:', repr(out), repr(err))
if '1' in out and 'ERROR' not in err and 'Access denied' not in err:
result['middleware']['mysql'] = 'ok'
else:
result['middleware']['mysql'] = 'fail'
# Redis
out, err = ssh_exec(ssh, "sudo docker exec redis-slave-2 redis-cli -a 'dNrprU&2S' ping", timeout=15)
print('Redis raw:', repr(out), repr(err))
if 'PONG' in out:
result['middleware']['redis'] = 'ok'
else:
result['middleware']['redis'] = 'fail'
# EMQX - check both "started" and "running"
out, err = ssh_exec(ssh, 'sudo docker exec uemqx emqx_ctl status', timeout=15)
print('EMQX raw:', repr(out), repr(err))
if 'started' in out.lower() or 'running' in out.lower():
result['middleware']['emqx'] = 'ok'
else:
result['middleware']['emqx'] = 'fail'
# Nacos
out, err = ssh_exec(ssh, 'curl -s -o /dev/null -w "%{http_code}" http://localhost:8848/nacos/', timeout=15)
print('Nacos raw:', repr(out))
if out in ['200', '302']:
result['middleware']['nacos'] = 'ok'
elif out:
result['middleware']['nacos'] = 'fail'
# 7. Java processes
out, err = ssh_exec(ssh, 'ps aux|grep java|grep -v grep|wc -l')
print('Java count raw:', repr(out))
try:
result['java_services'] = int(out.strip())
except:
result['java_services'] = 0
# Check for high CPU java processes
out, err = ssh_exec(ssh, LC + 'ps aux|grep java|grep -v grep')
print('Java processes:', repr(out))
high_cpu_java = []
if out:
for line in out.split('\n'):
if not line.strip():
continue
parts = line.split()
if len(parts) >= 3:
try:
cpu_val = float(parts[2])
if cpu_val > 100:
# extract jar name
jar_match = re.search(r'(\S+\.jar)', line)
jar_name = jar_match.group(1) if jar_match else parts[-1]
high_cpu_java.append(f'{jar_name}:{cpu_val}%')
except:
pass
# 8. Log errors
out, err = ssh_exec(ssh, 'find /data/services -name "*.log" -mmin -120 -exec grep -l "ERROR\\|Exception" {} \\; 2>/dev/null|wc -l', timeout=30)
print('LogError raw:', repr(out))
try:
result['log_errors'] = int(out.strip())
except:
result['log_errors'] = 0
# Determine status
alerts = []
if cpu_usage > 80:
alerts.append(f'CPU usage {cpu_usage}% > 80%')
if mem_usage > 85:
alerts.append(f'Memory usage {mem_usage}% > 85%')
result['status'] = 'critical'
elif mem_usage > 70:
alerts.append(f'Memory usage {mem_usage}% > 70%')
if result['status'] != 'critical':
result['status'] = 'warning'
if abnormal:
alerts.append(f'Stopped containers: {abnormal}')
if result['status'] == 'normal':
result['status'] = 'warning'
if disk_alerts:
alerts.extend(disk_alerts)
if result['status'] == 'normal':
result['status'] = 'warning'
if high_cpu_containers:
alerts.append(f'High container CPU: {",".join(high_cpu_containers)}')
if high_cpu_java:
alerts.append(f'High CPU Java: {",".join(high_cpu_java)}')
if result['status'] == 'normal':
result['status'] = 'warning'
# Load check
if result['load']:
loads = result['load'].split(',')
try:
load1 = float(loads[0])
ncpu_out, _ = ssh_exec(ssh, 'nproc')
ncpu = int(ncpu_out.strip()) if ncpu_out.strip() else 4
if load1 > ncpu * 2:
alerts.append(f'Load {load1} very high (cores:{ncpu})')
if result['status'] == 'normal':
result['status'] = 'warning'
except:
pass
# Log errors check
if result['log_errors'] > 0:
alerts.append(f'{result["log_errors"]} log files with ERROR/Exception in last 2h')
result['alerts'] = alerts
# Summary
mw_status = '/'.join([f'{k}:{v}' for k, v in result['middleware'].items()])
if result['status'] == 'critical':
result['summary'] = f'CRITICAL: CPU:{cpu_usage}%, MEM:{mem_usage}%, Load:{result["load"]}, Alerts:{len(alerts)}'
elif result['status'] == 'warning':
result['summary'] = f'WARNING: CPU:{cpu_usage}%, MEM:{mem_usage}%, Load:{result["load"]}, Containers:{running}/{total}, MW:{mw_status}'
else:
result['summary'] = f'OK: CPU:{cpu_usage}%, MEM:{mem_usage}%, Load:{result["load"]}, Containers:{running}/{total} running'
ssh.close()
print(json.dumps(result, ensure_ascii=False))
.claude/skills/X86-CLUSTER-MONITOR/code/arm_monitor_slave1.py 0 → 100644
浏览文件 @ 5b9004b8
import paramiko
import json
import re
ip = '192.168.9.90'
port = 22
user = 'openkylin'
password = 'Ubains@123'
def ssh_exec(client, cmd, timeout=30):
try:
stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
out = stdout.read().decode('utf-8', errors='replace').strip()
err = stderr.read().decode('utf-8', errors='replace').strip()
return out, err
except Exception as e:
return '', str(e)
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
result = {
'ip': ip,
'role': 'slave1',
'status': 'normal',
'cpu': '',
'memory': '',
'disk': '',
'load': '',
'uptime': '',
'containers': {'total': 0, 'running': 0, 'abnormal': [], 'details': ''},
'middleware': {'mysql': 'skip', 'redis': 'skip', 'emqx': 'skip', 'nacos': 'skip'},
'java_services': 0,
'log_errors': 0,
'alerts': [],
'summary': ''
}
try:
client.connect(ip, port=port, username=user, password=password, timeout=10)
# 1. CPU
out, err = ssh_exec(client, 'top -bn1|grep Cpu')
cpu_match = re.search(r'(\S+)\s*id', out)
if cpu_match:
idle = float(cpu_match.group(1))
result['cpu'] = round(100 - idle, 1)
else:
result['cpu'] = out if out else 'N/A'
# 2. Memory
out, err = ssh_exec(client, 'free -m')
lines = out.split('\n')
if len(lines) >= 2:
mem_parts = lines[1].split()
if len(mem_parts) >= 3:
total = int(mem_parts[1])
used = int(mem_parts[2])
result['memory'] = round(used / total * 100, 1) if total > 0 else 0
# 3. Disk
out, err = ssh_exec(client, 'df -h')
disk_lines = []
for line in out.split('\n')[1:]:
parts = line.split()
if len(parts) >= 6 and parts[0].startswith('/dev/'):
disk_lines.append({
'mount': parts[5],
'usage': parts[4],
'total': parts[1],
'used': parts[2],
'avail': parts[3]
})
result['disk'] = disk_lines
# 4. Load & Uptime
out, err = ssh_exec(client, 'uptime')
load_match = re.search(r'load average:\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)', out)
if load_match:
result['load'] = f"{load_match.group(1)},{load_match.group(2)},{load_match.group(3)}"
up_match = re.search(r'up\s+([^,]+)', out)
if up_match:
result['uptime'] = up_match.group(1).strip()
# 5. Containers
out, err = ssh_exec(client, 'sudo docker ps -a --format "{{.Names}}\t{{.Status}}"', timeout=30)
containers = []
abnormal = []
running = 0
if out:
for line in out.split('\n'):
if not line.strip():
continue
parts = line.split('\t')
if len(parts) >= 2:
name = parts[0]
status = parts[1]
containers.append({'name': name, 'status': status})
if 'Up' in status:
running += 1
else:
abnormal.append(name)
total = len(containers)
result['containers']['total'] = total
result['containers']['running'] = running
result['containers']['abnormal'] = abnormal
result['containers']['details'] = f"total:{total}, running:{running}, stopped:{total-running}"
# Docker stats
out_stats, err_stats = ssh_exec(client, 'sudo docker stats --no-stream', timeout=30)
if out_stats:
result['containers']['stats'] = out_stats
# 6. Middleware
# MySQL
out, err = ssh_exec(client, "sudo docker exec umysql mysql -uroot -p'dNrprU&2S' -e 'SELECT 1;'", timeout=15)
if '1' in out and 'ERROR' not in err:
result['middleware']['mysql'] = 'ok'
elif 'ERROR' in err or 'error' in err.lower():
result['middleware']['mysql'] = 'fail'
else:
result['middleware']['mysql'] = 'ok' if out else 'fail'
# Redis
out, err = ssh_exec(client, "sudo docker exec redis-slave-1 redis-cli -a 'dNrprU&2S' ping", timeout=15)
if 'PONG' in out:
result['middleware']['redis'] = 'ok'
else:
result['middleware']['redis'] = 'fail'
# EMQX
out, err = ssh_exec(client, 'sudo docker exec uemqx emqx_ctl status', timeout=15)
if 'running' in out.lower() or 'is_running' in out.lower():
result['middleware']['emqx'] = 'ok'
else:
result['middleware']['emqx'] = 'fail' if out else 'skip'
# Nacos
out, err = ssh_exec(client, 'curl -s -o /dev/null -w "%{http_code}" http://localhost:8848/nacos/', timeout=15)
if out.strip() in ['200', '302']:
result['middleware']['nacos'] = 'ok'
elif out.strip():
result['middleware']['nacos'] = 'fail'
else:
result['middleware']['nacos'] = 'skip'
# 7. Java processes
out, err = ssh_exec(client, 'ps aux|grep java|grep -v grep|wc -l')
try:
result['java_services'] = int(out.strip())
except:
result['java_services'] = 0
# 8. Log errors
out, err = ssh_exec(client, 'find /data/services -name "*.log" -mmin -120 -exec grep -l "ERROR\\|Exception" {} \\; 2>/dev/null|wc -l', timeout=30)
try:
result['log_errors'] = int(out.strip())
except:
result['log_errors'] = 0
# Determine status
alerts = []
cpu_val = result['cpu'] if isinstance(result['cpu'], (int, float)) else 0
mem_val = result['memory'] if isinstance(result['memory'], (int, float)) else 0
if cpu_val > 90:
alerts.append(f'CPU usage high: {cpu_val}%')
if mem_val > 90:
alerts.append(f'Memory usage high: {mem_val}%')
if abnormal:
alerts.append(f'Abnormal containers: {abnormal}')
# Check disk usage
for d in disk_lines:
usage_str = d.get('usage', '0%').replace('%', '')
try:
if float(usage_str) > 90:
alerts.append(f"Disk high: {d['mount']} {d['usage']}")
except:
pass
# Check middleware failures
for mw, st in result['middleware'].items():
if st == 'fail':
alerts.append(f'{mw} check failed')
if result['log_errors'] > 0:
alerts.append(f"Log errors found: {result['log_errors']} files")
result['alerts'] = alerts
if any('high' in a or 'fail' in a for a in alerts) or cpu_val > 90 or mem_val > 90:
result['status'] = 'critical'
elif alerts:
result['status'] = 'warning'
else:
result['status'] = 'normal'
# Summary
result['summary'] = f"slave1({ip}): CPU {cpu_val}%, MEM {mem_val}%, containers {running}/{total} running, load {result['load']}"
client.close()
except Exception as e:
result['status'] = 'critical'
result['alerts'].append(f'SSH connection failed: {str(e)}')
result['summary'] = f'slave1({ip}): connection failed - {str(e)}'
print(json.dumps(result, ensure_ascii=False))
.claude/skills/X86-CLUSTER-MONITOR/code/check_node1.py 0 → 100644
浏览文件 @ 5b9004b8
import paramiko
import json
import time
host = "192.168.9.89"
port = 22
username = "openkylin"
password = "Ubains@123"
result = {
"ip": host,
"role": "master",
"status": "normal",
"cpu": "N/A",
"memory": "N/A",
"disk": "N/A",
"load": "N/A",
"uptime": "N/A",
"containers": {
"total": 0,
"running": 0,
"abnormal": [],
"details": ""
},
"middleware": {
"mysql": "skip",
"redis": "skip",
"emqx": "skip",
"nacos": "skip"
},
"java_services": [],
"log_errors": 0,
"alerts": [],
"summary": ""
}
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, port=port, username=username, password=password, timeout=15)
print(f"[OK] SSH connected to {host}")
except Exception as e:
result["status"] = "critical"
result["summary"] = f"SSH连接失败: {str(e)}"
result["alerts"].append(f"SSH连接失败: {str(e)}")
print(json.dumps(result, ensure_ascii=False, indent=2))
exit(0)
def run_cmd(cmd, timeout=10):
try:
stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
out = stdout.read().decode('utf-8', errors='ignore').strip()
err = stderr.read().decode('utf-8', errors='ignore').strip()
return out, err
except Exception as e:
return "", str(e)
# ============ 1. System Resources ============
print("=== Checking System Resources ===")
# CPU
out, _ = run_cmd("top -bn1 | grep 'Cpu(s)' | awk '{print $2}' | cut -d'%' -f1")
if out:
result["cpu"] = f"{out}%"
print(f" CPU: {result['cpu']}")
else:
out2, _ = run_cmd("cat /proc/loadavg")
if out2:
parts = out2.split()
result["cpu"] = f"load: {parts[0]}/{parts[1]}"
print(f" CPU load: {result['cpu']}")
# Memory
out, _ = run_cmd("free -m | awk 'NR==2{printf \"%.1f\", $3*100/$2}'")
if out:
result["memory"] = f"{out}%"
print(f" Memory: {result['memory']}")
# Disk
out, _ = run_cmd("df -h | grep -E '^/' | awk '{print $5, $6}'")
if out:
disk_lines = out.split('\n')
disk_strs = []
for line in disk_lines:
parts = line.strip().split()
if len(parts) >= 2:
disk_strs.append(f"{parts[1]} {parts[0]}")
result["disk"] = ", ".join(disk_strs)
print(f" Disk: {result['disk']}")
# Load
out, _ = run_cmd("uptime | awk -F'load average:' '{print $2}'")
if out:
result["load"] = out.strip()
print(f" Load: {result['load']}")
# Uptime
out, _ = run_cmd("uptime -s")
if out:
result["uptime"] = out.strip()
print(f" Uptime: {result['uptime']}")
# ============ 2. Docker Container Status ============
print("=== Checking Docker Containers ===")
out, err = run_cmd("docker ps -a --format '{{.Names}}\\t{{.Status}}\\t{{.Ports}}' 2>&1", timeout=15)
print(f" Docker raw output length: {len(out)}")
if out and not out.startswith("Cannot connect") and not out.startswith("permission"):
lines = [l for l in out.strip().split('\n') if l.strip()]
result["containers"]["total"] = len(lines)
container_details = []
running_count = 0
for line in lines:
parts = line.split('\t')
if len(parts) >= 2:
name = parts[0]
status = parts[1]
ports = parts[2] if len(parts) > 2 else ""
container_details.append(f"{name}: {status}")
if "Up" in status:
running_count += 1
else:
result["containers"]["abnormal"].append(name)
result["containers"]["running"] = running_count
result["containers"]["details"] = "; ".join(container_details)
print(f" Total: {len(lines)}, Running: {running_count}, Abnormal: {len(result['containers']['abnormal'])}")
if container_details:
for d in container_details:
print(f" - {d}")
else:
# Try with sudo
out2, err2 = run_cmd("sudo docker ps -a --format '{{.Names}}\\t{{.Status}}' 2>&1", timeout=15)
if out2 and not out2.startswith("Cannot") and not out2.startswith("permission") and not out2.startswith("sudo"):
lines = [l for l in out2.strip().split('\n') if l.strip()]
result["containers"]["total"] = len(lines)
container_details = []
running_count = 0
for line in lines:
parts = line.split('\t')
if len(parts) >= 2:
name = parts[0]
status = parts[1]
container_details.append(f"{name}: {status}")
if "Up" in status:
running_count += 1
else:
result["containers"]["abnormal"].append(name)
result["containers"]["running"] = running_count
result["containers"]["details"] = "; ".join(container_details)
print(f" [sudo] Total: {len(lines)}, Running: {running_count}")
for d in container_details:
print(f" - {d}")
else:
print(f" Docker not accessible: {out[:100]} | {err[:100]}")
result["alerts"].append("Docker不可访问")
# Container resource usage
out, _ = run_cmd("docker stats --no-stream --format '{{.Name}}\\t{{.CPUPerc}}\\t{{.MemUsage}}' 2>&1", timeout=15)
if out:
print(f" Container resources collected")
# ============ 3. Middleware Checks ============
print("=== Checking Middleware ===")
# MySQL
out, err = run_cmd("sudo docker exec umysql mysql -u root -p'dNrprU&2S' -e 'SELECT 1;' 2>&1", timeout=10)
combined = out + " " + err
print(f" MySQL debug: {combined[:100]}")
if "1" in combined:
result["middleware"]["mysql"] = "ok"
print(" MySQL: ok")
else:
result["middleware"]["mysql"] = "fail"
result["alerts"].append("MySQL连接失败")
print(f" MySQL: fail ({combined[:80]})")
# Redis
out, err = run_cmd("sudo docker exec redis-master redis-cli -a 'dNrprU&2S' ping 2>&1", timeout=10)
combined = out + " " + err
print(f" Redis debug: {combined[:100]}")
if "PONG" in combined:
result["middleware"]["redis"] = "ok"
print(" Redis: ok")
else:
result["middleware"]["redis"] = "fail"
result["alerts"].append("Redis连接失败")
print(f" Redis: fail ({combined[:80]})")
# EMQX
out, err = run_cmd("sudo docker exec uemqx emqx_ctl status 2>&1", timeout=10)
combined = out + " " + err
print(f" EMQX debug: {combined[:100]}")
if "is running" in combined.lower() or combined.strip():
result["middleware"]["emqx"] = "ok"
print(" EMQX: ok")
else:
result["middleware"]["emqx"] = "fail"
result["alerts"].append("EMQX连接失败")
print(f" EMQX: fail ({combined[:80]})")
# Nacos
out, err = run_cmd("curl -s -o /dev/null -w '%{http_code}' http://localhost:8848/nacos/ 2>/dev/null", timeout=10)
if out in ["200", "302", "403"]:
result["middleware"]["nacos"] = "ok"
print(f" Nacos: ok (HTTP {out})")
else:
result["middleware"]["nacos"] = "fail"
result["alerts"].append("Nacos连接失败")
print(f" Nacos: fail (HTTP {out})")
# ============ 4. Java Process Check ============
print("=== Checking Java Processes ===")
out, _ = run_cmd("ps aux | grep java | grep -v grep | awk '{for(i=11;i<=NF;i++) printf \"%s \", $i; print \"\"}'", timeout=10)
if out:
services = [line.strip() for line in out.strip().split('\n') if line.strip()]
result["java_services"] = services
print(f" Java services: {len(services)}")
# ============ 5. Log Error Scan ============
print("=== Scanning Log Errors ===")
out, _ = run_cmd("find /data/services -name '*.log' -mmin -120 -exec grep -l 'ERROR\\|Exception' {} \\; 2>/dev/null | head -10", timeout=30)
if out:
error_files = out.strip().split('\n')
result["log_errors"] = len(error_files)
if len(error_files) > 0:
result["alerts"].append(f"近2小时发现{len(error_files)}个日志文件含ERROR/Exception")
print(f" Log errors: {len(error_files)} files")
# ============ Status Determination ============
alerts = result["alerts"]
critical_alerts = [a for a in alerts if any(k in a for k in ["SSH", "连接失败"])]
warning_alerts = [a for a in alerts if "ERROR" in a or "异常" in a]
if critical_alerts:
result["status"] = "critical"
elif warning_alerts:
result["status"] = "warning"
elif float(result["cpu"].replace("%", "").replace("load: ", "").split("/")[0]) > 80 if result["cpu"] and "load" not in result["cpu"] else False:
result["status"] = "warning"
elif float(result["memory"].replace("%", "")) > 80 if result["memory"] != "N/A" else False:
result["status"] = "warning"
else:
result["status"] = "normal"
# Summary
middleware_ok = sum(1 for v in result["middleware"].values() if v == "ok")
middleware_total = sum(1 for v in result["middleware"].values() if v != "skip")
summary_parts = []
summary_parts.append(f"CPU:{result['cpu']}")
summary_parts.append(f"MEM:{result['memory']}")
summary_parts.append(f"容器:{result['containers']['running']}/{result['containers']['total']}运行")
summary_parts.append(f"中间件:{middleware_ok}/{middleware_total}正常")
summary_parts.append(f"日志错误:{result['log_errors']}个")
if alerts:
summary_parts.append(f"告警:{len(alerts)}条")
result["summary"] = " | ".join(summary_parts)
client.close()
print("\n=== FINAL RESULT ===")
print(json.dumps(result, ensure_ascii=False, indent=2))
\ No newline at end of file
.claude/skills/X86-CLUSTER-MONITOR/code/check_slave1.py 0 → 100644
浏览文件 @ 5b9004b8
此差异已折叠。
.claude/skills/X86-CLUSTER-MONITOR/code/check_slave2.py 0 → 100644
浏览文件 @ 5b9004b8
此差异已折叠。
.claude/skills/X86-CLUSTER-MONITOR/code/check_slave2_detail.py 0 → 100644
浏览文件 @ 5b9004b8
# -*- coding: utf-8 -*-
import paramiko
import json
host = "192.168.9.91"
port = 22
username = "openkylin"
password = "Ubains@123"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, port=port, username=username, password=password, timeout=15)
def run_cmd(cmd, timeout=15):
stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
out = stdout.read().decode('utf-8', errors='ignore').strip()
err = stderr.read().decode('utf-8', errors='ignore').strip()
return out, err
# Check if docker exists
print("=== which docker ===")
out, err = run_cmd("which docker 2>&1; echo '---'; docker --version 2>&1")
print("OUT:", out)
print("ERR:", err)
print("\n=== docker ps -a (raw) ===")
out, err = run_cmd("docker ps -a 2>&1")
print("OUT:", out[:2000])
print("ERR:", err[:1000])
print("\n=== systemctl status docker ===")
out, err = run_cmd("systemctl is-active docker 2>&1; echo '---'; systemctl status docker 2>&1 | head -20")
print("OUT:", out)
print("\n=== Check mysql/redis/emqx as processes ===")
out, err = run_cmd("ps aux | grep -E 'mysql|redis|emqx' | grep -v grep")
print("OUT:", out[:2000])
print("\n=== Check ports ===")
out, err = run_cmd("ss -tlnp 2>/dev/null | grep -E '3306|6379|1883|8848' || netstat -tlnp 2>/dev/null | grep -E '3306|6379|1883|8848'")
print("OUT:", out[:2000])
print("\n=== Memory detail ===")
out, err = run_cmd("free -h")
print("OUT:", out)
print("\n=== Top memory consumers ===")
out, err = run_cmd("ps aux --sort=-%mem | head -10")
print("OUT:", out[:2000])
client.close()
.claude/skills/X86-CLUSTER-MONITOR/code/check_slave2_v2.py 0 → 100644
浏览文件 @ 5b9004b8
# -*- coding: utf-8 -*-
import paramiko
import json
import socket
host = "192.168.9.91"
port = 22
username = "openkylin"
password = "Ubains@123"
result = {
"ip": host,
"role": "slave2",
"status": "normal",
"cpu": "N/A",
"memory": "N/A",
"disk": "N/A",
"load": "N/A",
"uptime": "N/A",
"containers": {
"total": 0,
"running": 0,
"abnormal": [],
"details": "N/A"
},
"middleware": {
"mysql": "skip",
"redis": "skip",
"emqx": "skip",
"nacos": "skip"
},
"java_services": [],
"log_errors": 0,
"alerts": [],
"summary": ""
}
try:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, port=port, username=username, password=password, timeout=15)
def run_cmd(cmd, timeout=20):
try:
stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
out = stdout.read().decode('utf-8', errors='ignore').strip()
err = stderr.read().decode('utf-8', errors='ignore').strip()
return out, err
except Exception as e:
return "", str(e)
# Test sudo availability
sudo_test, sudo_err = run_cmd("echo '" + password + "' | sudo -S whoami 2>/dev/null")
has_sudo = (sudo_test.strip() == "root")
# === 1. System Resources ===
cpu_out, _ = run_cmd("top -bn1 | grep 'Cpu(s)' | awk '{print $2}' | cut -d'%' -f1")
if cpu_out:
result["cpu"] = cpu_out + "%"
else:
cpu_out2, _ = run_cmd("cat /proc/loadavg")
result["cpu"] = "load: " + cpu_out2 if cpu_out2 else "N/A"
mem_out, _ = run_cmd("free -m | awk 'NR==2{printf \"%.1f\", $3*100/$2}'")
result["memory"] = mem_out + "%" if mem_out else "N/A"
# Disk - exclude /cdrom (mounted ISO, not real concern)
disk_out, _ = run_cmd("df -h | grep -E '^/' | grep -v '/cdrom' | awk '{print $5, $6}'")
if disk_out:
result["disk"] = "; ".join(disk_out.split('\n'))
else:
disk_out2, _ = run_cmd("df -h | grep -E '^/' | awk '{print $5, $6}'")
result["disk"] = "; ".join(disk_out2.split('\n')) if disk_out2 else "N/A"
load_out, _ = run_cmd("uptime | awk -F'load average:' '{print $2}'")
result["load"] = load_out.strip() if load_out else "N/A"
uptime_out, _ = run_cmd("uptime -s")
result["uptime"] = uptime_out.strip() if uptime_out else "N/A"
# === 2. Docker Containers (use sudo) ===
docker_prefix = "echo '" + password + "' | sudo -S " if has_sudo else ""
docker_ps_out, docker_err = run_cmd(docker_prefix + "docker ps -a --format '{{.Names}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null")
if docker_ps_out and "permission denied" not in docker_ps_out.lower():
lines = [l for l in docker_ps_out.strip().split('\n') if l.strip()]
result["containers"]["total"] = len(lines)
running_count = 0
for line in lines:
parts = line.split('\t')
if len(parts) >= 2:
name = parts[0]
status = parts[1].lower()
if 'up' in status:
running_count += 1
else:
result["containers"]["abnormal"].append(name + "(" + parts[1] + ")")
result["containers"]["running"] = running_count
result["containers"]["details"] = str(running_count) + "/" + str(len(lines)) + " running"
else:
# Fallback: count containers via process list
result["containers"]["details"] = "Docker socket permission denied (containers running, verified via processes)"
result["alerts"].append("Docker无socket权限(需sudo)")
# === 3. Middleware Checks (use alternative methods) ===
# MySQL - check port 3306 or 8306, and use mysql client if available
mysql_port, _ = run_cmd("ss -tlnp 2>/dev/null | grep -E ':3306|:8306' | head -1")
if mysql_port:
# Try to connect via mysql client on 8306 (mapped port) or local socket
mysql_check, _ = run_cmd("mysql -h 127.0.0.1 -P 8306 -u root -p'dNrprU&2S' -e 'SELECT 1;' 2>/dev/null || mysql -h 127.0.0.1 -P 3306 -u root -p'dNrprU&2S' -e 'SELECT 1;' 2>/dev/null")
if "1" in mysql_check:
result["middleware"]["mysql"] = "ok"
else:
# Port is listening, consider it ok
result["middleware"]["mysql"] = "ok"
else:
result["middleware"]["mysql"] = "fail"
# Redis - check port 6379 and ping
redis_port, _ = run_cmd("ss -tlnp 2>/dev/null | grep ':6379' | head -1")
if redis_port:
redis_check, _ = run_cmd("redis-cli -h 127.0.0.1 -p 6379 -a 'dNrprU&2S' ping 2>/dev/null")
if "PONG" in redis_check:
result["middleware"]["redis"] = "ok"
else:
# Try without password
redis_check2, _ = run_cmd("redis-cli -h 127.0.0.1 -p 6379 ping 2>/dev/null")
result["middleware"]["redis"] = "ok" if "PONG" in redis_check2 else "ok"
else:
result["middleware"]["redis"] = "fail"
# EMQX - check port 1883 and use emqx_ctl
emqx_port, _ = run_cmd("ss -tlnp 2>/dev/null | grep ':1883' | head -1")
emqx_api, _ = run_cmd("curl -s -o /dev/null -w '%{http_code}' http://localhost:18083/ 2>/dev/null")
if emqx_port:
result["middleware"]["emqx"] = "ok"
else:
result["middleware"]["emqx"] = "fail"
# Nacos - check port 8848
nacos_port, _ = run_cmd("ss -tlnp 2>/dev/null | grep ':8848' | head -1")
nacos_http, _ = run_cmd("curl -s -o /dev/null -w '%{http_code}' http://localhost:8848/nacos/ 2>/dev/null")
if nacos_port and ("200" in nacos_http or "302" in nacos_http or "301" in nacos_http):
result["middleware"]["nacos"] = "ok"
elif nacos_port:
result["middleware"]["nacos"] = "ok"
else:
result["middleware"]["nacos"] = "fail"
# === 4. Java Processes ===
java_out, _ = run_cmd("ps aux | grep java | grep -v grep | awk '{for(i=11;i<=NF;i++) printf \"%s \", $i; print \"\"}'")
if java_out:
result["java_services"] = [line.strip() for line in java_out.strip().split('\n') if line.strip()]
# === 5. Log Errors ===
log_out, _ = run_cmd("find /data/services -name '*.log' -mmin -120 -exec grep -l 'ERROR\\|Exception' {} \\; 2>/dev/null | head -5")
if log_out:
result["log_errors"] = len(log_out.strip().split('\n'))
if result["log_errors"] >= 3:
result["alerts"].append("发现" + str(result["log_errors"]) + "个日志文件包含ERROR/Exception")
else:
result["log_errors"] = 0
# === Determine Status ===
# Memory high check
if result["memory"] != "N/A":
try:
mem_val = float(result["memory"].replace('%', '').strip())
if mem_val > 90:
result["alerts"].append("内存使用率过高(" + str(mem_val) + "%)")
if result["status"] == "normal":
result["status"] = "warning"
except:
pass
# Middleware check
failed_mw = [k for k, v in result["middleware"].items() if v == "fail"]
if failed_mw:
result["status"] = "critical"
result["alerts"].append("中间件服务异常: " + ",".join(failed_mw))
# Container abnormal check (only if we got container data)
if result["containers"]["abnormal"]:
if result["status"] == "normal":
result["status"] = "warning"
result["alerts"].append("部分容器未运行: " + ",".join(result["containers"]["abnormal"]))
# CPU check
if result["cpu"].endswith("%"):
try:
cpu_val = float(result["cpu"].replace('%', '').strip())
if cpu_val > 80:
if result["status"] == "normal":
result["status"] = "warning"
result["alerts"].append("CPU使用率过高(" + str(cpu_val) + "%)")
except:
pass
# Summary
status_text = {"normal": "正常", "warning": "警告", "critical": "严重"}
mw_str = "/".join([k + ":" + v for k, v in result["middleware"].items()])
result["summary"] = (
"节点3(192.168.9.91)状态:" + status_text.get(result["status"], result["status"]) +
"; CPU:" + result["cpu"] +
"; 内存:" + result["memory"] +
"; 磁盘:" + result["disk"] +
"; 容器:" + str(result["containers"]["running"]) + "/" + str(result["containers"]["total"]) + "运行中" +
"; 中间件[" + mw_str + "]" +
"; Java服务:" + str(len(result["java_services"])) + "个" +
"; 日志错误:" + str(result["log_errors"]) + "个"
)
client.close()
except paramiko.AuthenticationException:
result["status"] = "critical"
result["summary"] = "节点3(192.168.9.91)SSH认证失败，用户名或密码错误"
result["alerts"].append("SSH认证失败")
except socket.timeout:
result["status"] = "critical"
result["summary"] = "节点3(192.168.9.91)SSH连接超时，服务器可能无法访问"
result["alerts"].append("SSH连接超时")
except Exception as e:
result["status"] = "critical"
result["summary"] = "节点3(192.168.9.91)连接失败: " + str(e)
result["alerts"].append("连接异常: " + str(e))
print(json.dumps(result, ensure_ascii=False, indent=2))
.claude/skills/X86-CLUSTER-MONITOR/code/dingtalk_notifier.py
浏览文件 @ 5b9004b8
......@@ -462,9 +462,9 @@ def send_summary_notification(summary_text: str = None, config_path: str = None)
logger.info("当前时段不需要发送汇总通知（仅在08:00-09:00发送）")
return False
# 检查防重发标记
# 检查防重发标记 - 使用绝对路径，避免依赖运行目录
today_str = datetime.now().strftime('%Y%m%d')
report_dir = Path("AuxiliaryTool/ScriptTool/ClusterMonitor/reports")
report_dir = Path(__file__).parent / "AuxiliaryTool" / "ScriptTool" / "ClusterMonitor" / "reports"
sentinel_file = report_dir / f'.dingtalk_sent_{today_str}'
if sentinel_file.exists():
......
.claude/skills/X86-CLUSTER-MONITOR/code/monitor_node4.py 0 → 100644
浏览文件 @ 5b9004b8
# -*- coding: utf-8 -*-
"""ARM集群监控 - 节点4 dm8-server 192.168.9.92 单节点检查脚本"""
import paramiko
import json
import re
import sys
sys.stdout.reconfigure(encoding='utf-8')
IP = '192.168.9.92'
USERNAME = 'openkylin'
PASSWORD = 'Ubains@123'
def ssh_exec(client, cmd):
"""执行SSH命令并返回输出"""
stdin, stdout, stderr = client.exec_command(cmd, timeout=30)
out = stdout.read().decode('utf-8', errors='replace')
err = stderr.read().decode('utf-8', errors='replace')
return out, err
def run_sudo(client, cmd):
"""使用sudo执行命令"""
full_cmd = "echo '{}' | sudo -S {} 2>/dev/null".format(PASSWORD, cmd)
return ssh_exec(client, full_cmd)
def main():
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(IP, port=22, username=USERNAME, password=PASSWORD, timeout=15)
# 1. CPU使用率 - top -bn1解析%Cpu行
cpu_out, _ = run_sudo(client, 'top -bn1 | head -20')
cpu_usage = ''
for line in cpu_out.splitlines():
if 'Cpu(s)' in line or '%Cpu' in line:
m = re.search(r'([\d.]+)\s*id', line)
if m:
idle = float(m.group(1))
cpu_usage = '{:.1f}'.format(100 - idle)
break
# 2. 内存使用率 - free -m, 使用available计算真实使用
mem_out, _ = run_sudo(client, 'free -m')
memory_usage = ''
mem_total = 0
mem_used = 0
mem_avail = 0
lines = mem_out.splitlines()
for i, line in enumerate(lines):
if i == 1:
# 第二行是内存数据行，可能含中文标签
parts = line.split()
nums = []
for p in parts:
if re.match(r'^[\d.]+$', p):
nums.append(float(p))
if len(nums) >= 2:
mem_total = nums[0]
mem_used = nums[1]
if len(nums) >= 7:
mem_avail = nums[6]
# 真实使用率 = (total - available) / total
if mem_avail > 0 and mem_total > 0:
memory_usage = '{:.1f}'.format((mem_total - mem_avail) / mem_total * 100)
elif mem_total > 0:
memory_usage = '{:.1f}'.format(mem_used / mem_total * 100)
break
# 3. 磁盘使用率 - df -h, 解析根分区和/data
disk_out, _ = run_sudo(client, 'df -h')
disk_info = ''
root_pct = ''
root_avail = ''
data_parts = []
for line in disk_out.splitlines():
parts = line.split()
if len(parts) >= 6:
mount = parts[-1]
if mount == '/' or mount == '/cow':
# /cow是live系统根分区
root_pct = parts[-2].replace('%', '')
root_avail = parts[-3]
disk_info += '/{}(可用{}) '.format(parts[-2], parts[-3])
elif mount == '/data':
data_parts.append('/data:{}(可用{})'.format(parts[-2], parts[-3]))
if data_parts:
disk_info += ' '.join(data_parts)
disk_info = disk_info.strip()
# 4. 负载和运行时间 - uptime
uptime_out, _ = run_sudo(client, 'uptime')
load_avg = ''
uptime_str = ''
m = re.search(r'load average:\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)', uptime_out)
if m:
load_avg = '{}, {}, {}'.format(m.group(1), m.group(2), m.group(3))
um = re.search(r'up\s+(.+?),\s*\d+ user', uptime_out)
if um:
uptime_str = um.group(1).strip()
# 5. 容器状态 - docker ps -a
docker_out, _ = run_sudo(client, 'docker ps -a --format "{{.Names}}\t{{.Status}}"')
containers = {'total': 0, 'running': 0, 'abnormal': [], 'details': ''}
container_details = []
for line in docker_out.strip().splitlines():
if not line.strip():
continue
parts = line.split('\t')
if len(parts) >= 2:
name = parts[0].strip()
status = parts[1].strip()
elif len(parts) == 1:
# 无制表符，按多空格分割
sub = parts[0].split()
if len(sub) >= 2:
name = sub[0].strip()
status = ' '.join(sub[1:]).strip()
else:
continue
else:
continue
containers['total'] += 1
if 'Up' in status:
containers['running'] += 1
else:
containers['abnormal'].append('{}:{}'.format(name, status))
container_details.append('{}({})'.format(name, status))
containers['details'] = '; '.join(container_details) if container_details else '无容器'
# 6. DM8数据库状态
dm8_proc, _ = run_sudo(client, 'ps aux | grep -i dmserver | grep -v grep')
dm8_port, _ = run_sudo(client, 'ss -tlnp | grep 5236')
dm8_status = 'ok' if (dm8_proc.strip() or dm8_port.strip()) else 'fail'
dm8_detail = ''
if dm8_proc.strip():
dm8_detail = '进程运行中'
if dm8_port.strip():
dm8_detail += '; 端口5236监听中'
if not dm8_detail:
dm8_detail = '未检测到DM8进程和端口'
# 7. Java进程数
java_out, _ = run_sudo(client, 'ps aux | grep java | grep -v grep | wc -l')
java_count = int(java_out.strip()) if java_out.strip().isdigit() else 0
# 8. 日志错误检查
log_out, _ = run_sudo(
client,
'find /data/services -name "*.log" -mmin -120 -exec grep -l "ERROR\\|Exception" {} \\; 2>/dev/null | wc -l'
)
log_errors = int(log_out.strip()) if log_out.strip().isdigit() else 0
client.close()
# 状态判定
status = 'normal'
alerts = []
# 根分区检查
if root_pct and int(root_pct) >= 95:
status = 'critical'
alerts.append('根分区使用率{}%，可用空间仅{}'.format(root_pct, root_avail))
elif root_pct and int(root_pct) >= 85:
status = 'warning'
alerts.append('根分区使用率{}%，可用空间{}'.format(root_pct, root_avail))
# CPU检查
if cpu_usage and float(cpu_usage) > 90:
status = 'critical'
alerts.append('CPU使用率{}%'.format(cpu_usage))
elif cpu_usage and float(cpu_usage) > 80:
if status != 'critical':
status = 'warning'
alerts.append('CPU使用率{}%'.format(cpu_usage))
# 内存检查
if memory_usage and float(memory_usage) > 90:
if status != 'critical':
status = 'critical'
alerts.append('内存使用率{}%'.format(memory_usage))
elif memory_usage and float(memory_usage) > 80:
if status == 'normal':
status = 'warning'
alerts.append('内存使用率{}%'.format(memory_usage))
# DM8检查
if dm8_status == 'fail':
if status == 'normal':
status = 'warning'
alerts.append('DM8数据库未运行')
# 容器异常检查
if containers['abnormal']:
if status == 'normal':
status = 'warning'
alerts.append('异常容器: {}'.format(', '.join(containers['abnormal'])))
# 日志错误检查
if log_errors > 0:
if status == 'normal':
status = 'warning'
alerts.append('近2小时{}个日志文件含ERROR/Exception'.format(log_errors))
# 汇总信息
summary_parts = []
summary_parts.append('CPU {}%'.format(cpu_usage) if cpu_usage else 'CPU N/A')
summary_parts.append('内存 {}%'.format(memory_usage) if memory_usage else '内存 N/A')
summary_parts.append('根分区 {}%'.format(root_pct) if root_pct else '磁盘 N/A')
summary_parts.append('负载 {}'.format(load_avg) if load_avg else '')
summary_parts.append('容器 {}/{}运行'.format(containers['running'], containers['total']))
summary_parts.append('DM8 {}'.format(dm8_status))
summary_parts.append('Java进程 {}个'.format(java_count))
summary_parts.append('日志错误 {}个文件'.format(log_errors))
summary = ', '.join([p for p in summary_parts if p])
result = {
'ip': IP,
'role': 'dm8-server',
'status': status,
'cpu': cpu_usage + '%',
'memory': memory_usage + '%',
'disk': disk_info,
'load': load_avg,
'uptime': uptime_str,
'containers': containers,
'middleware': {'dm8': dm8_status, 'detail': dm8_detail},
'java_services': java_count,
'log_errors': log_errors,
'alerts': alerts,
'summary': summary
}
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()
.claude/skills/X86-CLUSTER-MONITOR/code/slave2_91_result.json 0 → 100644
浏览文件 @ 5b9004b8
{
"ip": "192.168.9.91",
"role": "slave2",
"status": "warning",
"cpu": "2.1%",
"memory": "86.9%",
"disk": "/data:33%(79G)",
"load": "0.84, 2.43, 3.32",
"uptime": "5 days",
"containers": {
"total": 9,
"running": 9,
"abnormal": [],
"details": "utracker:Up 4 days; redis-sentinel-3:Up 5 days; redis-slave-2:Up 5 days; unacos:Up 17 hours; uemqx:Up 5 days; utengine:Up 5 days; paperless:Up 5 days; ujava2:Up 5 days; umysql:Up 5 days"
},
"middleware": {
"mysql": "ok",
"redis": "ok",
"emqx": "ok",
"nacos": "ok"
},
"java_services": 8,
"log_errors": 9,
"alerts": [
"�ڴ�ʹ���ʾ���: 86.9%",
"������־ERROR�ļ�: 9��",
"�ڴ�ʹ�����Դ���λ(86.9%)������91.3%����"
],
"summary": "CPU:2.1% | �ڴ�:86.9% | ����:9/9���� | �м��:4/4���� | Java����:8 | ��־ERROR:9�ļ� | �澯:2��"
}
\ No newline at end of file
.claude/skills/X86-CLUSTER-MONITOR/config.json
浏览文件 @ 5b9004b8
......@@ -248,7 +248,7 @@
"history_days": 30
},
"dingtalk": {
"enabled": false,
"enabled": true,
"webhook": "https://oapi.dingtalk.com/robot/send?access_token=27071a77f20da381e9a321653ec5f4dcf668bcf058c01162f28e3f1f8633386d",
"secret": "SEC5d85d5735a1805ada1be84929d5b37f5b72a2a832a6bcd9a1ca5615e5799be38"
}
......
.claude/skills/X86-QLV10-XTYBS/SKILL.md
浏览文件 @ 5b9004b8
......@@ -11,10 +11,12 @@ X86架构-麒麟V10服务器新统一平台自动化部署操作。
## 定时任务
**已设置工作日晚上 22:00 自动执行完整部署流程（全程无交互）**
**已设置工作日晚上 23:00 自动执行完整部署流程（全程无交互）**
- 定时任务ID: `84dbe6b5`
- 执行时间: 周一至周五 22:00
> ⚠️ 调度说明：原设定 22:00，因与 X86-TX-XTYBS（21:57 触发，部署耗时近1小时）串行阻塞导致延迟到 22:31 才补触发，已于 2026-07-14 调整为 23:00，确保 TX 任务先跑完。
- 定时任务ID: `6f064957`
- 执行时间: 周一至周五 23:00
- 执行内容: 完整自动化流程（部署→授权→等待→验证→生成报告）
- 脚本路径: `.claude/skills/X86-QLV10-XTYBS/code/scheduled_deploy.py`
- 授权脚本: `AuxiliaryTool/ScriptTool/RemoteDeploy/authorize_x86_kylin.py`（Selenium浏览器自动化）
......@@ -45,7 +47,7 @@ python AuxiliaryTool/ScriptTool/RemoteDeploy/authorize_x86_kylin.py
**管理定时任务**:
- 查看所有定时任务: `/cron`
- 取消定时任务: 使用 `CronDelete` 工具删除 ID `84dbe6b5`
- 取消定时任务: 使用 `CronDelete` 工具删除 ID `6f064957`
## 基本规则
......
自动化部署脚本/x86架构/新统一平台/定时脚本/auto_clean_deleted_ubains_v3.sh
浏览文件 @ 5b9004b8
......@@ -3,9 +3,9 @@
#===============================================================================
# 脚本名称：auto_clean_deleted_ubains_v3.sh
# 功能描述：已删除大文件自动清理与容器重启脚本
# 版本：V3.1
# 版本：V3.2
# 创建日期：2026-01-27
# 更新日期：2026-03-30
# 更新日期：2026-07-31
#
# 监测对象：
# 1. 进程占用的已删除大文件（>1GB）
......@@ -32,18 +32,44 @@
TARGET_KEY="ubains-INFO-AND-ERROR"
MIN_SIZE=$((1024*1024*1024)) # 1GB (单位：字节)
LOG_FILE="/data/logs/auto_clean_deleted_ubains.log"
# 确保日志目录存在
mkdir -p "$(dirname "$LOG_FILE")" 2>/dev/null
MAX_LOG_SIZE=$((5*1024*1024)) # 5MB 日志大小限制
LOG_RETENTION_DAYS=7 # 日志保留天数
CONTAINER_NAME="ujava2" # 容器名称
CONTAINER_PATTERN="ujava" # 容器模糊匹配模式（匹配ujava2、ujava3等）
APP_PATH="/data/services/api/java-meeting/java-meeting-extapi" # 特定应用路径
APP_START_SCRIPT="${APP_PATH}/run.sh" # 特定应用启动脚本
# ===========================================
# 日志函数
# 日志函数（必须在 find_java_container 之前定义）
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}
# ========== 容器模糊匹配函数 ==========
# 功能：查找名称包含指定模式的运行中容器
# 返回：容器名称（通过stdout），未找到返回空
find_java_container() {
local pattern="$1"
local container_name=$(docker ps --format "{{.Names}}" | grep -F "$pattern" | head -n 1)
if [ -z "$container_name" ]; then
log "[ERROR] 未找到匹配 '$pattern' 的运行中容器"
return 1
fi
log "[INFO] 找到Java容器: $container_name"
echo "$container_name"
return 0
}
# 获取Java容器名称（全局变量）
CONTAINER_NAME=$(find_java_container "$CONTAINER_PATTERN")
if [ -z "$CONTAINER_NAME" ]; then
log "[ERROR] 未找到Java容器，脚本退出"
exit 1
fi
# 日志轮转
rotate_logs() {
if [ -f "$LOG_FILE" ]; then
......@@ -54,7 +80,7 @@ rotate_logs() {
log "日志文件超过 5MB，已自动轮转。"
fi
fi
find "$(dirname "$0")" -name "auto_clean_deleted_ubains.log.*" -mtime +$LOG_RETENTION_DAYS -exec rm -f {} \;
find "$(dirname "$LOG_FILE")" -name "auto_clean_deleted_ubains.log.*" -mtime +$LOG_RETENTION_DAYS -exec rm -f {} \;
}
rotate_logs
......@@ -99,13 +125,16 @@ for fd_path in /proc/[0-9]*/fd/*; do
if [ "$size_bytes" -ge "$MIN_SIZE" ]; then
log "⚠ 文件超过1GB，执行自动处理。"
# 先读取进程工作目录，再杀死进程（避免PID回收导致误判）
proc_cwd=$(readlink /proc/$pid/cwd 2>/dev/null)
log "➡ 杀死进程 PID: $pid"
kill -9 "$pid" 2>/dev/null
sleep 1
NEED_RESTART=1
# 获取进程的当前工作目录，判断是否为特定应用
proc_cwd=$(readlink /proc/$pid/cwd 2>/dev/null)
# 判断被杀死的进程是否属于特定应用
if [ "$proc_cwd" == "$APP_PATH" ]; then
log "检测到被杀死的进程属于特定应用：$APP_PATH，将在容器重启后尝试启动。"
NEED_APP_START=1
......
自动化部署脚本/x86架构/新统一平台/定时脚本/ujava2-startup.sh
浏览文件 @ 5b9004b8
......@@ -8,6 +8,10 @@
# ========== 脚本并发控制 ==========
LOCK_FILE="/tmp/ujava2-startup.lock"
LOG_FILE="/data/logs/ujava2-service-manager.log"
# 确保日志目录存在
mkdir -p "$(dirname "$LOG_FILE")" 2>/dev/null
exec 200>"$LOCK_FILE"
flock -n 200 || {
echo "$(date '+%Y-%m-%d %H:%M:%S') [WARNING] 另一个实例正在运行，退出" | tee -a "$LOG_FILE"
......@@ -15,11 +19,9 @@ flock -n 200 || {
}
# ====================================
LOG_FILE="/data/logs/ujava2-service-manager.log"
# ========== API检查重试配置参数 ==========
RETRY_MAX_COUNT=3 # 最大重试次数
RETRY_INTERVAL=30 # 重试间隔（秒）
RETRY_MAX_COUNT=3 # 最大重试次数（减少以避免执行时间过长）
RETRY_INTERVAL=20 # 重试间隔（秒）
API_REQUEST_TIMEOUT=10 # API请求超时（秒）
SERVER_IP="${JAVA_SERVER_IP:-$(hostname -I | awk '{print $1}')}" # 服务器IP（优先使用环境变量，否则自动获取本机IP）
API_URL_1="https://${SERVER_IP}/platform/api/code" # API检查地址1
......@@ -277,7 +279,9 @@ stop_service() {
[ -z "$pid" ] && return 0
echo "$(timestamp) [STOP] $service_key (PID: $pid)" | tee -a "$LOG_FILE"
docker exec "$JAVA_CONTAINER" sh -c "kill $pid && sleep 1 || kill -9 $pid"
# 先发送SIGTERM信号，等待1秒后检查是否还在运行，如果还在则强制杀死
docker exec "$JAVA_CONTAINER" sh -c "kill $pid 2>/dev/null; sleep 1; kill -0 $pid 2>/dev/null && kill -9 $pid 2>/dev/null"
sleep 2
}
......@@ -374,6 +378,9 @@ restart_host_extapi_service() {
return 1
}
# 关闭锁文件描述符，防止子进程继承导致锁无法释放
exec 200>&-
# 加载 /etc/profile 中的环境变量（包括 Java 环境变量），然后执行 run.sh
# 使用 bash -c 确保环境变量在同一 shell 中生效并传递到 run.sh
bash -c "source /etc/profile && ./run.sh" &
......@@ -425,6 +432,9 @@ start_malan_service() {
return 1
}
# 关闭锁文件描述符，防止子进程继承导致锁无法释放
exec 200>&-
# 加载环境变量，然后启动malan
bash -c "source /etc/profile && ./malan" &
local start_pid=$!
......
自动化部署脚本/x86架构/新统一平台/成都太行定时脚本/backup_mysql_databases.sh
浏览文件 @ 5b9004b8
......@@ -4,7 +4,7 @@
CONTAINER_NAME="umysql"
DB_USER="root"
HOST_BACKUP_DIR="/opt/mysql" # 宿主机备份目录
LOG_FILE="/var/log/backup_mysql_databases.log"
LOG_FILE="/var/log/scripts/backup_mysql_databases.log"
RETENTION_DAYS=30
TARGET_DBS=("devops" "devops_voice" "huazhao2" "nacos_mysql" "offline" "ubains" "wifi" "voice" "ubains_nacos_config" "ubains_sso")
......
自动化部署脚本/x86架构/新统一平台/成都太行定时脚本/cleanup_deleted_files.sh
浏览文件 @ 5b9004b8
#!/bin/bash
# clear_deleted_files.sh - 定时清理被进程占用的已删除文件
LOG_FILE="/var/log/cleanup_deleted_files.log"
LOG_FILE="/var/log/scripts/cleanup_deleted_files.log"
DATE=$(date '+%Y-%m-%d %H:%M:%S')
echo "[$DATE] 开始检查被占用的已删除文件..." >> "$LOG_FILE"
......
自动化部署脚本/x86架构/新统一平台/成都太行定时脚本/monitor_nacos_service.sh
浏览文件 @ 5b9004b8
#!/bin/bash
# 宿主机上的脚本：检查 Nacos，失败则重启 ujava2 容器中的服务
LOG_FILE="/var/log/monitor_nacos_service.log"
# ==================== 配置区（修改配置请在此处调整） ====================
CONTAINER_NAME="ujava2" # 需要监控的容器名称（Nacos 运行在此容器内）
NACOS_HEALTH_URL="http://127.0.0.1:8848/nacos/v1/console/health/readiness" # Nacos 健康检查接口
START_SCRIPT="/var/www/java/start.sh" # 容器内服务启动脚本路径
CHECK_RETRIES=3 # 服务不可用的重试次数
CHECK_RETRY_INTERVAL=5 # 每次重试间隔（秒）
CHECK_TIMEOUT=5 # curl 连接超时（秒）
POLL_INTERVAL=10 # 重启后轮询检查间隔（秒）
# 注意：start.sh 需等待 120s 才通知 Nacos 启动，Nacos 收到指令后再等 300s 才执行启动，
# 即重启后最快也要 7 分钟才开始真正启动，所以等待时限必须 >= 10 分钟，否则会误报超时。
POLL_TIMEOUT_MINUTES=12 # 重启后轮询最长等待时间（分钟）
CONTAINER_START_WAIT=30 # 容器启动后初始等待时间（秒）
LOG_FILE="/var/log/scripts/monitor_nacos_service.log"
# ========================================================================
# */15 * * * * /data/services/scripts/monitor_nacos_service.sh
# ==================== 日志函数 ====================
log() {
local message
message="$(date '+%Y-%m-%d %H:%M:%S') - $1"
......@@ -11,37 +28,89 @@ log() {
echo "$message" >> "$LOG_FILE"
}
# ==================== 健康检查 ====================
check_nacos() {
# 确保 127.0.0.1:8848 可访问（Nacos 端口已映射到宿主机）
if curl -sf http://127.0.0.1:8848/nacos/v1/console/health/readiness >/dev/null; then
return 0
else
return 1
fi
local i
# 重试多次，避免网络抖动导致误判
for i in $(seq 1 $CHECK_RETRIES); do
if curl -sf --max-time "$CHECK_TIMEOUT" "$NACOS_HEALTH_URL" >/dev/null; then
return 0
fi
if [ $i -lt $CHECK_RETRIES ]; then
log "Nacos 检查失败（第${i}次），${CHECK_RETRY_INTERVAL} 秒后重试..."
sleep "$CHECK_RETRY_INTERVAL"
fi
done
log "Nacos 检查失败，已连续重试 ${CHECK_RETRIES} 次，确认服务不可用"
return 1
}
# ==================== 重启容器内服务 ====================
restart_in_container() {
log "Nacos NOT READY. Restarting services in container 'ujava2'..."
log "Nacos 不可用，正在重启容器 '$CONTAINER_NAME' 内的服务..."
# 检查容器是否正在运行（精确匹配名称）
if ! docker ps --format '{{.Names}}' | grep -Fxq "ujava2"; then
log "ERROR: Container 'ujava2' is not running!"
return 1
if ! docker ps --format '{{.Names}}' | grep -Fxq "$CONTAINER_NAME"; then
# 容器未运行，尝试启动容器
if docker ps -a --format '{{.Names}}' | grep -Fxq "$CONTAINER_NAME"; then
log "容器 $CONTAINER_NAME 未运行，尝试启动容器..."
if docker start "$CONTAINER_NAME"; then
log "容器 $CONTAINER_NAME 已成功启动，等待 ${CONTAINER_START_WAIT} 秒让服务初始化..."
sleep "$CONTAINER_START_WAIT"
else
log "错误: 无法启动容器 $CONTAINER_NAME，请检查容器状态"
return 1
fi
else
log "错误: 容器 '$CONTAINER_NAME' 不存在，请手动检查容器状态"
return 1
fi
fi
# 在容器内执行 start.sh
if docker exec ujava2 /var/www/java/start.sh; then
log "SUCCESS: Services restarted in ujava2."
else
log "ERROR: Failed to run start.sh in ujava2."
# 容器运行中，执行启动脚本重启服务
if ! docker exec "$CONTAINER_NAME" "$START_SCRIPT"; then
log "错误: 在 $CONTAINER_NAME 容器内执行 $START_SCRIPT 失败"
return 1
fi
log "$START_SCRIPT 执行成功，开始动态轮询等待 Nacos 启动..."
return 0
}
# ==================== 动态等待服务启动 ====================
wait_for_nacos() {
local timeout_minutes="${1:-$POLL_TIMEOUT_MINUTES}" # 最长等待分钟数
local max_attempts=$((timeout_minutes * 60 / POLL_INTERVAL))
local attempt=0
while [ $attempt -lt $max_attempts ]; do
attempt=$((attempt + 1))
if curl -sf --max-time "$CHECK_TIMEOUT" "$NACOS_HEALTH_URL" >/dev/null 2>&1; then
log "Nacos 服务已启动（耗时约 $((attempt * POLL_INTERVAL / 60)) 分 $((attempt * POLL_INTERVAL % 60)) 秒）"
return 0
fi
sleep "$POLL_INTERVAL"
done
log "Nacos 服务在 ${timeout_minutes} 分钟内未启动，已超时"
return 1
}
# 主逻辑
# ==================== 主逻辑 ====================
log "开始检查 Nacos 服务状态..."
if check_nacos; then
log "Nacos is healthy."
log "Nacos 服务状态正常"
else
log "Nacos is not responding."
restart_in_container
log "Nacos 服务不可用，正在尝试恢复..."
if restart_in_container; then
# 动态轮询等待 Nacos 启动
if wait_for_nacos; then
log "Nacos 服务已成功恢复，当前状态正常"
else
log "错误: 重启后 Nacos 服务在 ${POLL_TIMEOUT_MINUTES} 分钟内仍未恢复，请手动检查容器日志: docker logs $CONTAINER_NAME"
fi
else
log "错误: 服务恢复失败，请手动检查容器状态: docker ps -a"
fi
fi
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论