diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c9bbb39 --- /dev/null +++ b/.env.example @@ -0,0 +1,37 @@ +HOST='0.0.0.0' +PORT=8081 +SqlName="db.sql" + +# Redis 配置 +REDIS_HOST="127.0.0.1:6379" +REDIS_PASSWORD= +LOG_LEVEL="debug" +LOG_SAVE="false" +LOG_SAVE_PATH="./logs" + +# ------------------ Nacos 配置 ------------------ +NACOS_HOSTS=nacos1.example.com,nacos2.example.com,nacos3.example.com +NACOS_PORT=8848 +NACOS_NAMESPACE= +NACOS_GROUP_NAME=DEFAULT_GROUP +NACOS_USER= +NACOS_PASSWORD= + +# ------------------ 日志上报 (httplog → Redis → ES) ------------------ +ES_REDIS_KEY=access_log +ES_BULK_URL=https://elasticsearch.hostidc.net/_bulk +ES_INDEX_PREFIX=access +ES_USERNAME= +ES_PASSWORD= +ES_BATCH_SIZE=1000 +ES_POLL_INTERVAL_MS=200 +ES_HTTP_TIMEOUT_MS=5000 + +# ------------------ GRPC 配置 ------------------ +GRPC_TOKEN= + +# ------------------ 微服务注册 ------------------ +NACOS_SERVICE_NAME= +NACOS_SERVICE_PORT= +NACOS_SERVICE_HOST= +NACOS_SERVICE_WEIGHT=10 diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 8dcfbe1..def817c 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -1,4 +1,4 @@ -name: 'Build ApiServer' +name: 'Build & Deploy' on: push: tags: @@ -14,16 +14,16 @@ jobs: - name: Build Action run: | - go build -o quantumProfit ./cmd/main_program - go build -o cliControl ./cmd/cli_control + go build -ldflags="-s -w" -o server ./cmd/main_program + go build -ldflags="-s -w" -o cli ./cmd/cli_control - name: Save artifact uses: actions/upload-artifact@v3 with: - name: quantumProfit + name: build-artifacts path: | - ./quantumProfit - ./cliControl + ./server + ./cli deploy: needs: build @@ -32,16 +32,14 @@ jobs: - name: Download Artifact uses: actions/download-artifact@v3 with: - name: quantumProfit + name: build-artifacts - name: Set up SSH run: | mkdir -p ~/.ssh echo "${{ secrets.PUBLICT_PRIVATE_KEY }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - # 将服务器列表写入临时文件 echo "${{ vars.DEPLOY_SERVER_LIST }}" > server_list.txt - # 读取文件并为每个服务器设置 SSH while read -r ip; do if [ -n "$ip" ]; then ssh-keyscan -H "$ip" >> ~/.ssh/known_hosts @@ -50,15 +48,17 @@ jobs: - name: Deploy to servers run: | - # 读取临时文件并循环部署 while read -r ip; do if [ -n "$ip" ]; then echo "Deploying to $ip..." - scp -o StrictHostKeyChecking=no quantumProfit ${{ vars.ROOT_USER_NAME }}@"$ip":/root/quantumProfit.tmp - scp -o StrictHostKeyChecking=no cliControl ${{ vars.ROOT_USER_NAME }}@"$ip":/root/cliControl.tmp - ssh -n ${{ vars.ROOT_USER_NAME }}@"$ip" "mv /root/quantumProfit.tmp /root/quantumProfit" < /dev/null - ssh -n ${{ vars.ROOT_USER_NAME }}@"$ip" "mv /root/cliControl.tmp /root/cliControl" < /dev/null - ssh -n ${{ vars.ROOT_USER_NAME }}@"$ip" "cd /root && bash ./chmodFile.sh" < /dev/null + scp -o StrictHostKeyChecking=no server ${{ vars.ROOT_USER_NAME }}@"$ip":/root/server.tmp + scp -o StrictHostKeyChecking=no cli ${{ vars.ROOT_USER_NAME }}@"$ip":/root/cli.tmp + ssh -n ${{ vars.ROOT_USER_NAME }}@"$ip" << 'ENDSSH' + mv /root/server.tmp /root/server + mv /root/cli.tmp /root/cli + chmod +x /root/server /root/cli + systemctl restart server cli + ENDSSH echo "Deployment to $ip completed" fi - done < server_list.txt \ No newline at end of file + done < server_list.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b2e814 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +.env +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.test +*.out +*.log + +/vendor/ +/logs/ +/tmp/ + +__debug_bin* +.idea/ +.vscode/ +*.swp +*.swo diff --git a/README.md b/README.md index 42c3259..53be7b8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,295 @@ # apiServer 微服务模板 -## 使用步骤 +基于 [CloudWego Hertz](https://github.com/cloudwego/hertz) 的 Go 微服务脚手架,集成 Nacos 服务注册/发现 + gRPC 客户端 + 访问日志上报(Redis → ES)。 -- 拉取该项目仓库 -- 修改配置文件 .env -- 修改项目名称,将项目文件中的 apiServer_service 替换为项目名称 -- 新建gitea仓库,修改本地仓库地址 -- 项目入口在 cmd/main_program 文件下 \ No newline at end of file +## 项目结构 + +``` +├── apps/ # 业务处理层 (Handler) +├── cmd/ +│ ├── main_program/ # 主程序入口 (HTTP 服务) +│ └── cli_control/ # CLI 工具入口 (httplog 上报) +├── middleware/ # HTTP 中间件 (Recovery, AccessLog, CORS) +├── models/request_models/ # 请求参数模型 +├── proto/ # Protobuf 生成代码 +├── routes/ # 路由定义 +├── utils/ +│ ├── httplog/ # HTTP 访问日志采集 & ES 上报 +│ ├── logger/ # 日志工具 (logrus) +│ ├── nacos/ # Nacos 服务注册/发现/配置 +│ ├── redis_tools/ # Redis 连接 & 通用操作 +│ ├── request/ # 请求绑定 & 统一响应 +│ └── server_cli/ # gRPC 客户端 +├── start.sh / stop.sh / restart.sh +├── .env.example # 环境变量示例 +└── go.mod +``` + +## 快速开始 + +```bash +# 1. 复制配置 +cp .env.example .env +# 编辑 .env 填入实际配置 + +# 2. 安装依赖 +go mod tidy + +# 3. 开发运行 +go run ./cmd/main_program # HTTP 服务 +go run ./cmd/cli_control # httplog 上报 + +# 4. 构建 +go build -ldflags="-s -w" -o server ./cmd/main_program +go build -ldflags="-s -w" -o cli ./cmd/cli_control +``` + +## 模板使用步骤 + +1. 拉取该项目仓库 +2. 复制 `.env.example` 为 `.env` 并修改配置 +3. 全局替换 `apiServer_service` 为你的项目模块名 +4. 修改 `go.mod` 中的 module 名称 +5. 新建 Gitea 仓库,修改本地仓库地址 + +--- + +## 模块说明 + +### cmd/main_program — HTTP 主服务 + +启动 Hertz HTTP 服务,绑定路由和中间件,可选注册到 Nacos。 + +```bash +go run ./cmd/main_program +``` + +启动流程:加载 `.env` → 校验 `HOST`/`PORT` → 注册中间件(Recovery、AccessLog、CORS)→ 绑定路由 → Nacos 注册(可选)→ 启动 HTTP 监听 → 等待信号优雅关闭。 + +### cmd/cli_control — httplog 日志上报 + +独立后台进程,从 Redis 队列中消费访问日志,批量写入 Elasticsearch。 + +```bash +go run ./cmd/cli_control +``` + +启动后会以轮询方式从 `ES_REDIS_KEY` 队列中批量 pop 日志条目,组装 ES `_bulk` 请求写入 `ES_INDEX_PREFIX-YYYY.MM.DD` 索引。 + +### middleware — HTTP 中间件 + +在 `cmd/main_program/routs.go` 中统一注册: + +```go +r.Use(middleware.Recovery()) // panic 恢复,防止单个请求崩溃整个服务 +r.Use(middleware.AccessLog()) // 请求日志(方法、路径、状态码、耗时) +r.Use(middleware.CORS()) // 跨域支持 +``` + +### utils/httplog — 访问日志采集 + +Hertz Tracer 实现,在请求完成后采集完整的访问事件(方法、路径、状态码、耗时、请求体、响应体等),通过 Redis List 异步缓冲。 + +**在主服务中接入:** + +```go +import ( + "apiServer_service/utils/httplog" + "apiServer_service/utils/redis_tools" + "github.com/cloudwego/hertz/pkg/app/server" +) + +rdb := redis_tools.ConnectRedis() +tracer := httplog.NewRedisAccessLogTracer(rdb, "access_log", "my-service", + httplog.WithSkipPrefix("/health"), // 跳过健康检查路径 + httplog.WithMaxResponseBody(4096), // 响应体最大采集 4KB + httplog.WithUserIDExtractor(func(c *app.RequestContext) uint { + // 根据你的认证方式提取 user_id + return 0 + }), +) +h := server.Default(server.WithTracer(tracer)) +``` + +**特性:** +- 敏感字段自动脱敏(password, token, secret 等) +- multipart 文件字段替换为 `[file]` 占位符 +- 非文本响应自动跳过(图片、zip 等) +- 异步写入 Redis,队列满时丢弃(不阻塞业务) + +### utils/redis_tools — Redis 工具 + +单例连接,提供通用 KV 和 List 操作: + +```go +import "apiServer_service/utils/redis_tools" + +// 连接(全局只初始化一次) +rdb := redis_tools.ConnectRedis() + +// KV 操作 +redis_tools.SetCache("key", "value", 10*time.Minute) +val, err := redis_tools.GetCache("key") +redis_tools.Del("key1", "key2") +redis_tools.Exists("key") + +// List 操作 +redis_tools.AddToList("queue", "item") +items, _ := redis_tools.GetAllFromList("queue") +redis_tools.RemoveFromList("queue", "item") +``` + +### utils/nacos — Nacos 服务注册/发现/配置 + +```go +import "apiServer_service/utils/nacos" + +// 注册当前服务(读取 NACOS_SERVICE_* 环境变量) +nacos.RegisterService() + +// 发现服务(带内存缓存) +instance, err := nacos.DiscoverService("user-service") +addr := instance.Ip + ":" + strconv.Itoa(int(instance.Port)) + +// 配置管理 +content := nacos.GetConfig("app.yaml", "DEFAULT_GROUP") +nacos.AddConfig("app.yaml", "DEFAULT_GROUP", "key: value") +``` + +### utils/server_cli — gRPC 客户端 + +通过 Nacos 服务发现获取 gRPC 地址,连接复用: + +```go +import "apiServer_service/utils/server_cli" + +err := server_cli.ReportVisit(token, note, ip, os, point, userId) +defer server_cli.CloseGrpcConn() +``` + +### utils/request — 请求绑定 & 统一响应 + +```go +import "apiServer_service/utils/request" + +// 参数绑定(失败自动返回 400) +var req MyRequest +if err := request.BindRequestStruct(c, &req); err != nil { + return +} + +// 统一响应 +request.Success(c, data) // 200 {"code":200,"message":"Success","data":...} +request.BadRequest(c, "参数错误") // 400 +request.Unauthorized(c, "未登录") // 401 +request.NotFound(c, "资源不存在") // 404 +request.Error(c, 500, "服务器内部错误") // 自定义状态码 +request.FileResponse(c, "/path/to/file", "download.zip") +``` + +### routes — 路由定义 + +在 `routes/` 下按模块拆分路由文件,在 `cmd/main_program/routs.go` 中注册: + +```go +func SetupRoutes(r *server.Hertz) { + r.Use(middleware.Recovery()) + r.Use(middleware.AccessLog()) + r.Use(middleware.CORS()) + + api := r.Group("/api") + { + routes.RegisterIndexRoutes(api) + // routes.RegisterUserRoutes(api) // 新增模块在此注册 + } +} +``` + +--- + +## 生产部署 + +### 构建 + +```bash +go build -ldflags="-s -w" -o server ./cmd/main_program +go build -ldflags="-s -w" -o cli ./cmd/cli_control +``` + +### 首次安装(systemd 服务注册) + +将二进制、`.env`、脚本和 `deploy/` 目录上传到服务器后执行: + +```bash +chmod +x install.sh start.sh stop.sh restart.sh server cli + +# 安装 systemd 服务 + 设置开机自启 +sudo bash install.sh +``` + +`install.sh` 会自动: +1. 将 `deploy/*.service` 适配当前路径后复制到 `/etc/systemd/system/` +2. 执行 `systemctl daemon-reload` +3. 执行 `systemctl enable server cli` 开机自启 + +### 日常运维 + +```bash +bash start.sh # 启动全部服务 +bash stop.sh # 停止全部服务 +bash restart.sh # 重启全部服务 +``` + +也可以直接使用 `systemctl` 管理单个服务: + +```bash +systemctl status server # 查看主服务状态 +systemctl status cli # 查看 CLI 状态 +systemctl restart server # 只重启主服务 +journalctl -u server -f # 查看主服务实时日志 +journalctl -u cli -f --since today # 查看 CLI 今日日志 +``` + +### systemd 服务特性 + +| 特性 | 说明 | +|------|------| +| 开机自启 | `install.sh` 执行后自动启用 | +| 崩溃自动重启 | `Restart=always`,server 间隔 3s,cli 间隔 5s | +| 优雅关闭 | `KillSignal=SIGTERM`,等待 10s 超时后 SIGKILL | +| 环境变量 | 通过 `EnvironmentFile` 加载 `.env` | +| 文件描述符 | `LimitNOFILE=65536` | +| 日志 | 同时写入 `logs/*.out` 和 `journalctl` | + +### 部署目录结构 + +``` +/root/ +├── server # HTTP 主服务二进制 +├── cli # httplog 上报二进制 +├── .env # 环境配置 +├── deploy/ +│ ├── server.service # systemd 服务单元(模板) +│ └── cli.service +├── install.sh # 首次安装脚本 +├── start.sh # 启动 +├── stop.sh # 停止 +├── restart.sh # 重启 +└── logs/ + ├── server.out + └── cli.out +``` + +## 内置功能清单 + +- Hertz HTTP 框架 + 路由分组 +- Recovery / AccessLog / CORS 中间件 +- 统一 JSON 响应格式 +- 参数绑定与校验 +- HTTP 访问日志采集 → Redis 缓冲 → ES 批量上报 +- Redis 工具(单例连接池) +- Nacos 服务注册、发现、配置管理 +- gRPC 客户端(含连接复用) +- 彩色日志输出 + 文件日志 +- 优雅关闭 (Graceful Shutdown) +- systemd 服务管理(开机自启 + 崩溃自动重启) diff --git a/apps/index_view.go b/apps/index_view.go index a54a591..7ce9255 100644 --- a/apps/index_view.go +++ b/apps/index_view.go @@ -2,17 +2,17 @@ package apps import ( "apiServer_service/models/request_models" - "apiServer_service/utils/loger" + "apiServer_service/utils/logger" "apiServer_service/utils/request" "context" "github.com/cloudwego/hertz/pkg/app" ) func Ping(ctx context.Context, c *app.RequestContext) { - var requests request_models.IndexRequest - if err := request.BindRequestStruct(c, &requests); err != nil { + var req request_models.IndexRequest + if err := request.BindRequestStruct(c, &req); err != nil { return } - loger.Info("Ping", requests.Name) + logger.Info("Ping", req.Name) request.Success(c, "pong") } diff --git a/cmd/cli_control/main.go b/cmd/cli_control/main.go index 6624431..0150f2f 100644 --- a/cmd/cli_control/main.go +++ b/cmd/cli_control/main.go @@ -1,19 +1,33 @@ package main import ( + "apiServer_service/utils/httplog" + "apiServer_service/utils/logger" "fmt" + "os" + "os/signal" + "syscall" + "github.com/joho/godotenv" - "log" ) func init() { - // 在 init 中加载 .env 文件 - err := godotenv.Load(".env") - if err != nil { - log.Fatal("Error loading .env file") + if err := godotenv.Load(".env"); err != nil { + fmt.Println("Warning: .env file not found, using system environment variables") } } func main() { - fmt.Println("子应用 main 方法") + redisKey := os.Getenv("ES_REDIS_KEY") + esIndexPrefix := os.Getenv("ES_INDEX_PREFIX") + + logger.Info("CLI", fmt.Sprintf("httplog 上报启动 (redis_key=%s, es_index=%s-*)", redisKey, esIndexPrefix)) + + go httplog.Updater(redisKey, esIndexPrefix) + + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + + logger.Info("CLI", "正在关闭...") } diff --git a/cmd/main_program/main.go b/cmd/main_program/main.go index 4e57d53..511d007 100644 --- a/cmd/main_program/main.go +++ b/cmd/main_program/main.go @@ -1,39 +1,57 @@ package main import ( + "apiServer_service/utils/logger" "apiServer_service/utils/nacos" + "fmt" + "os" + "os/signal" + "syscall" + "github.com/cloudwego/hertz/pkg/app/server" "github.com/joho/godotenv" - "log" - "os" ) func init() { - // 在 init 中加载 .env 文件 - err := godotenv.Load(".env") - if err != nil { - log.Fatal("Error loading .env file") + if err := godotenv.Load(".env"); err != nil { + fmt.Println("Warning: .env file not found, using system environment variables") } } func main() { host := os.Getenv("HOST") port := os.Getenv("PORT") + if host == "" || port == "" { + logger.Fatal("Config", "HOST 和 PORT 环境变量必须设置") + } + h := server.Default( - server.WithHostPorts(host + ":" + port), + server.WithHostPorts(host+":"+port), + server.WithExitWaitTime(0), ) SetupRoutes(h) - // 注册 nacos 服务 - err := nacos.RegisterService() - if err != nil { - log.Println("nacos register service error", err) + if os.Getenv("NACOS_SERVICE_NAME") != "" { + if err := nacos.RegisterService(); err != nil { + logger.Warn("Nacos", "服务注册失败: ", err) + } else { + logger.Info("Nacos", "服务注册成功") + } } - // 启动服务器 - err = h.Run() - if err != nil { - log.Fatal(err) - return + go func() { + h.Spin() + }() + + logger.Info("Server", fmt.Sprintf("服务启动于 %s:%s", host, port)) + + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + + logger.Info("Server", "正在关闭服务...") + if err := h.Close(); err != nil { + logger.Error("Server", "关闭失败: ", err) } + logger.Info("Server", "服务已停止") } diff --git a/cmd/main_program/routs.go b/cmd/main_program/routs.go index d1ed657..3a77e2b 100644 --- a/cmd/main_program/routs.go +++ b/cmd/main_program/routs.go @@ -1,14 +1,19 @@ package main import ( - "apiServer_service/routs" + "apiServer_service/middleware" + "apiServer_service/routes" + "github.com/cloudwego/hertz/pkg/app/server" ) func SetupRoutes(r *server.Hertz) { - auth := r.Group("/api") + r.Use(middleware.Recovery()) + r.Use(middleware.AccessLog()) + r.Use(middleware.CORS()) + + api := r.Group("/api") { - // index 路由 - routs.RegisterIndexRoutes(auth) + routes.RegisterIndexRoutes(api) } } diff --git a/deploy/cli.service b/deploy/cli.service new file mode 100644 index 0000000..653cf93 --- /dev/null +++ b/deploy/cli.service @@ -0,0 +1,22 @@ +[Unit] +Description=API Server CLI (httplog uploader) +After=network.target redis.service +Wants=redis.service + +[Service] +Type=simple +WorkingDirectory=/root +ExecStart=/root/cli +Restart=always +RestartSec=5 +LimitNOFILE=65536 +KillSignal=SIGTERM +TimeoutStopSec=10 + +EnvironmentFile=-/root/.env + +StandardOutput=append:/root/logs/cli.out +StandardError=append:/root/logs/cli.out + +[Install] +WantedBy=multi-user.target diff --git a/deploy/server.service b/deploy/server.service new file mode 100644 index 0000000..227b059 --- /dev/null +++ b/deploy/server.service @@ -0,0 +1,22 @@ +[Unit] +Description=API Server (HTTP) +After=network.target redis.service +Wants=redis.service + +[Service] +Type=simple +WorkingDirectory=/root +ExecStart=/root/server +Restart=always +RestartSec=3 +LimitNOFILE=65536 +KillSignal=SIGTERM +TimeoutStopSec=10 + +EnvironmentFile=-/root/.env + +StandardOutput=append:/root/logs/server.out +StandardError=append:/root/logs/server.out + +[Install] +WantedBy=multi-user.target diff --git a/go.mod b/go.mod index 90902e3..69ba443 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,10 @@ module apiServer_service -go 1.23rc1 +go 1.23 require ( github.com/cloudwego/hertz v0.9.6 + github.com/go-redis/redis/v8 v8.11.5 github.com/joho/godotenv v1.5.1 github.com/nacos-group/nacos-sdk-go/v2 v2.2.9 github.com/sirupsen/logrus v1.9.3 @@ -43,6 +44,7 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/netpoll v0.6.4 // indirect github.com/deckarep/golang-set v1.7.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..aa8e844 --- /dev/null +++ b/install.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -e + +APP_DIR="$(cd "$(dirname "$0")" && pwd)" +DEPLOY_DIR="$APP_DIR/deploy" +SERVICE_DIR="/etc/systemd/system" +LOG_DIR="$APP_DIR/logs" + +if [ "$(id -u)" -ne 0 ]; then + echo "请使用 root 权限运行: sudo bash install.sh" + exit 1 +fi + +mkdir -p "$LOG_DIR" + +echo "=== 安装 systemd 服务 ===" + +for svc in server cli; do + src="$DEPLOY_DIR/${svc}.service" + if [ ! -f "$src" ]; then + echo "[${svc}] service 文件不存在: $src" + continue + fi + + # 用实际路径替换模板中的 /root + sed "s|WorkingDirectory=/root|WorkingDirectory=$APP_DIR|g; \ + s|ExecStart=/root/|ExecStart=$APP_DIR/|g; \ + s|EnvironmentFile=-/root/.env|EnvironmentFile=-$APP_DIR/.env|g; \ + s|/root/logs/|$LOG_DIR/|g" \ + "$src" > "$SERVICE_DIR/${svc}.service" + + echo "[${svc}] 已安装到 $SERVICE_DIR/${svc}.service" +done + +systemctl daemon-reload + +for svc in server cli; do + systemctl enable "$svc" + echo "[${svc}] 已设置开机自启" +done + +echo "" +echo "=== 安装完成 ===" +echo "使用方式:" +echo " bash start.sh # 启动全部" +echo " bash stop.sh # 停止全部" +echo " bash restart.sh # 重启全部" +echo " systemctl status server # 查看主服务状态" +echo " journalctl -u server -f # 查看主服务实时日志" diff --git a/middleware/access_log.go b/middleware/access_log.go new file mode 100644 index 0000000..fc4416e --- /dev/null +++ b/middleware/access_log.go @@ -0,0 +1,27 @@ +package middleware + +import ( + "apiServer_service/utils/logger" + "context" + "fmt" + "time" + + "github.com/cloudwego/hertz/pkg/app" +) + +func AccessLog() app.HandlerFunc { + return func(ctx context.Context, c *app.RequestContext) { + start := time.Now() + c.Next(ctx) + latency := time.Since(start) + + logger.Info("HTTP", + fmt.Sprintf("%s %s %d %s", + string(c.Method()), + string(c.Request.URI().Path()), + c.Response.StatusCode(), + latency, + ), + ) + } +} diff --git a/middleware/cors.go b/middleware/cors.go new file mode 100644 index 0000000..82d393a --- /dev/null +++ b/middleware/cors.go @@ -0,0 +1,23 @@ +package middleware + +import ( + "context" + "net/http" + + "github.com/cloudwego/hertz/pkg/app" +) + +func CORS() app.HandlerFunc { + return func(ctx context.Context, c *app.RequestContext) { + c.Header("Access-Control-Allow-Origin", "*") + c.Header("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + c.Header("Access-Control-Allow-Headers", "Origin, Content-Type, Authorization") + c.Header("Access-Control-Max-Age", "86400") + + if string(c.Method()) == http.MethodOptions { + c.AbortWithStatus(http.StatusNoContent) + return + } + c.Next(ctx) + } +} diff --git a/middleware/recovery.go b/middleware/recovery.go new file mode 100644 index 0000000..e01dd17 --- /dev/null +++ b/middleware/recovery.go @@ -0,0 +1,24 @@ +package middleware + +import ( + "apiServer_service/utils/logger" + "apiServer_service/utils/request" + "context" + "fmt" + "runtime/debug" + + "github.com/cloudwego/hertz/pkg/app" +) + +func Recovery() app.HandlerFunc { + return func(ctx context.Context, c *app.RequestContext) { + defer func() { + if r := recover(); r != nil { + logger.Error("Panic Recovery", fmt.Sprintf("%v\n%s", r, debug.Stack())) + request.Error(c, 500, "Internal Server Error") + c.Abort() + } + }() + c.Next(ctx) + } +} diff --git a/restart.sh b/restart.sh new file mode 100644 index 0000000..1cd4bc1 --- /dev/null +++ b/restart.sh @@ -0,0 +1,10 @@ +#!/bin/bash +SERVICES="server cli" + +for svc in $SERVICES; do + systemctl restart "$svc" + echo "[$svc] 已重启" +done + +echo "" +systemctl status $SERVICES --no-pager -l diff --git a/routs/index_routs.go b/routes/index_routes.go similarity index 92% rename from routs/index_routs.go rename to routes/index_routes.go index 2453f29..ebe5db6 100644 --- a/routs/index_routs.go +++ b/routes/index_routes.go @@ -1,4 +1,4 @@ -package routs +package routes import ( "apiServer_service/apps" diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..665abe0 --- /dev/null +++ b/start.sh @@ -0,0 +1,14 @@ +#!/bin/bash +SERVICES="server cli" + +for svc in $SERVICES; do + if systemctl is-active --quiet "$svc"; then + echo "[$svc] 已在运行" + else + systemctl start "$svc" + echo "[$svc] 已启动" + fi +done + +echo "" +systemctl status $SERVICES --no-pager -l diff --git a/stop.sh b/stop.sh new file mode 100644 index 0000000..96bc207 --- /dev/null +++ b/stop.sh @@ -0,0 +1,11 @@ +#!/bin/bash +SERVICES="server cli" + +for svc in $SERVICES; do + if systemctl is-active --quiet "$svc"; then + systemctl stop "$svc" + echo "[$svc] 已停止" + else + echo "[$svc] 未在运行" + fi +done diff --git a/utils/httplog/redis_tracer.go b/utils/httplog/redis_tracer.go new file mode 100644 index 0000000..41524f9 --- /dev/null +++ b/utils/httplog/redis_tracer.go @@ -0,0 +1,285 @@ +package httplog + +import ( + "context" + "encoding/json" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/cloudwego/hertz/pkg/app" + "github.com/cloudwego/hertz/pkg/common/tracer/stats" + "github.com/go-redis/redis/v8" +) + +type AccessEvent struct { + Timestamp string `json:"@timestamp"` + TsMs int64 `json:"ts_ms"` + Service string `json:"service,omitempty"` + Instance string `json:"instance,omitempty"` + Method string `json:"method"` + Host string `json:"host,omitempty"` + Path string `json:"path"` + UserID uint `json:"user_id,omitempty"` + Body string `json:"body,omitempty"` + Query string `json:"query,omitempty"` + Result string `json:"result,omitempty"` + ResultTruncated bool `json:"result_truncated,omitempty"` + RequestURI string `json:"request_uri,omitempty"` + Route string `json:"route,omitempty"` + Status int `json:"status"` + CostMs int64 `json:"cost_ms"` + RecvBytes int `json:"recv_bytes"` + SendBytes int `json:"send_bytes"` + RemoteAddr string `json:"remote_addr,omitempty"` + ClientIP string `json:"client_ip,omitempty"` + UserAgent string `json:"ua,omitempty"` + Referer string `json:"referer,omitempty"` + RequestID string `json:"request_id,omitempty"` + Error string `json:"error,omitempty"` + Panicked bool `json:"panicked"` + PanicValue string `json:"panic_value"` +} + +// UserIDExtractor 从请求上下文中提取用户 ID 的函数签名。 +// 不同项目可根据自身认证方式实现此函数。 +type UserIDExtractor func(c *app.RequestContext) uint + +type RedisListWriter struct { + rdb *redis.Client + key string + ch chan []byte + flushInterval time.Duration + maxBatch int + + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + dropped uint64 +} + +func NewRedisListWriter(rdb *redis.Client, key string, queueSize int, maxBatch int, flushInterval time.Duration) *RedisListWriter { + ctx, cancel := context.WithCancel(context.Background()) + w := &RedisListWriter{ + rdb: rdb, + key: key, + ch: make(chan []byte, queueSize), + flushInterval: flushInterval, + maxBatch: maxBatch, + ctx: ctx, + cancel: cancel, + } + w.wg.Add(1) + go w.loop() + return w +} + +func (w *RedisListWriter) Enqueue(b []byte) { + select { + case w.ch <- b: + default: + atomic.AddUint64(&w.dropped, 1) + } +} + +func (w *RedisListWriter) Dropped() uint64 { return atomic.LoadUint64(&w.dropped) } + +func (w *RedisListWriter) Close(ctx context.Context) error { + w.cancel() + done := make(chan struct{}) + go func() { + w.wg.Wait() + close(done) + }() + select { + case <-ctx.Done(): + return ctx.Err() + case <-done: + return nil + } +} + +func (w *RedisListWriter) loop() { + defer w.wg.Done() + + ticker := time.NewTicker(w.flushInterval) + defer ticker.Stop() + + flush := func(buf [][]byte) { + if len(buf) == 0 { + return + } + args := make([]interface{}, 0, len(buf)) + for _, b := range buf { + args = append(args, string(b)) + } + cctx, cancel := context.WithTimeout(w.ctx, 2*time.Second) + _ = w.rdb.RPush(cctx, w.key, args...).Err() + cancel() + } + + var buf [][]byte + for { + select { + case <-w.ctx.Done(): + flush(buf) + return + case b := <-w.ch: + buf = append(buf, b) + if len(buf) >= w.maxBatch { + flush(buf) + buf = buf[:0] + } + case <-ticker.C: + flush(buf) + buf = buf[:0] + } + } +} + +type TracerOption func(*RedisAccessLogTracer) + +// WithUserIDExtractor 设置自定义的用户 ID 提取函数 +func WithUserIDExtractor(fn UserIDExtractor) TracerOption { + return func(t *RedisAccessLogTracer) { + t.userIDFn = fn + } +} + +// WithSkipPrefix 设置需要跳过日志记录的路径前缀(如 /health, /metrics) +func WithSkipPrefix(prefix string) TracerOption { + return func(t *RedisAccessLogTracer) { + t.skipPrefix = prefix + } +} + +// WithMaxResponseBody 设置响应体采集的最大字节数 +func WithMaxResponseBody(n int) TracerOption { + return func(t *RedisAccessLogTracer) { + t.maxRespBody = n + } +} + +type RedisAccessLogTracer struct { + writer *RedisListWriter + service string + instance string + skipPrefix string + maxRespBody int + userIDFn UserIDExtractor +} + +func NewRedisAccessLogTracer(rdb *redis.Client, redisKey string, service string, opts ...TracerOption) *RedisAccessLogTracer { + host, _ := os.Hostname() + t := &RedisAccessLogTracer{ + writer: NewRedisListWriter(rdb, redisKey, 8192, 512, 200*time.Millisecond), + service: service, + instance: host, + maxRespBody: 2048, + } + for _, opt := range opts { + opt(t) + } + return t +} + +func (t *RedisAccessLogTracer) Start(ctx context.Context, _ *app.RequestContext) context.Context { + return ctx +} + +func (t *RedisAccessLogTracer) Finish(_ context.Context, c *app.RequestContext) { + if t.skipPrefix != "" { + p := string(c.Request.URI().PathOriginal()) + if len(p) >= len(t.skipPrefix) && p[:len(t.skipPrefix)] == t.skipPrefix { + return + } + } + + ti := c.GetTraceInfo() + st := ti.Stats() + + var cost time.Duration + if rpcStart := st.GetEvent(stats.HTTPStart); rpcStart != nil { + if rpcFinish := st.GetEvent(stats.HTTPFinish); rpcFinish != nil { + cost = rpcFinish.Time().Sub(rpcStart.Time()) + } + } + + now := time.Now().UTC() + req := &c.Request + uri := req.URI() + + remoteAddr := "" + if ra := c.RemoteAddr(); ra != nil { + remoteAddr = ra.String() + } + + errStr := "" + if st.Error() != nil { + errStr = st.Error().Error() + } + panicked, panicVal := st.Panicked() + panicStr := "" + if panicVal != nil { + panicStr = anyToString(panicVal) + } + + ev := AccessEvent{ + Timestamp: now.Format(time.RFC3339Nano), + TsMs: now.UnixMilli(), + Service: t.service, + Instance: t.instance, + Method: string(req.Method()), + Host: string(req.Host()), + Path: string(uri.PathOriginal()), + Query: string(uri.QueryString()), + RequestURI: string(uri.RequestURI()), + Status: c.Response.StatusCode(), + CostMs: cost.Milliseconds(), + RecvBytes: st.RecvSize(), + SendBytes: st.SendSize(), + RemoteAddr: remoteAddr, + ClientIP: c.ClientIP(), + UserAgent: string(req.Header.UserAgent()), + Referer: string(req.Header.Peek("Referer")), + RequestID: string(req.Header.Peek("X-Request-Id")), + Error: errStr, + Panicked: panicked, + PanicValue: panicStr, + } + + if t.userIDFn != nil { + ev.UserID = t.userIDFn(c) + } + + if formJSON, ok, _ := FormBodyToJSONWithFilePlaceholder(c); ok { + ev.Body = string(formJSON) + } + + if respBody, ok, trunc := ResponseBodySnippet(c, t.maxRespBody); ok { + ev.Result = respBody + ev.ResultTruncated = trunc + } + + b, err := json.Marshal(ev) + if err != nil { + return + } + t.writer.Enqueue(b) +} + +func (t *RedisAccessLogTracer) Close(ctx context.Context) error { + return t.writer.Close(ctx) +} + +func anyToString(v interface{}) string { + switch x := v.(type) { + case string: + return x + default: + b, _ := json.Marshal(x) + return string(b) + } +} diff --git a/utils/httplog/request.go b/utils/httplog/request.go new file mode 100644 index 0000000..6279e74 --- /dev/null +++ b/utils/httplog/request.go @@ -0,0 +1,132 @@ +package httplog + +import ( + "encoding/json" + "mime" + "strings" + + "github.com/cloudwego/hertz/pkg/app" +) + +var sensitiveKeys = []string{"password", "secret", "token", "key", "passwd"} + +const redactedValue = "[REDACTED]" + +// desensitization:只对第一层 key 做脱敏,不递归 +func desensitization(data map[string]any) ([]byte, error) { + out := make(map[string]any, len(data)) + + for k, v := range data { + if isSensitiveKey(k) { + out[k] = redactedValue + } else { + out[k] = v + } + } + return json.Marshal(out) +} + +// 大小写不敏感、包含式匹配 +func isSensitiveKey(key string) bool { + k := strings.ToLower(key) + for _, s := range sensitiveKeys { + if strings.Contains(k, s) { + return true + } + } + return false +} + +// FormBodyToJSONWithFilePlaceholder : +// - 普通表单:完整转 JSON +// - multipart 文件字段:不取任何文件信息,只输出 "[file]" 或 ["[file]", ...] +func FormBodyToJSONWithFilePlaceholder(c *app.RequestContext) (jsonBytes []byte, ok bool, err error) { + ct := string(c.Request.Header.ContentType()) + mediaType, _, _ := mime.ParseMediaType(ct) + mediaType = strings.ToLower(mediaType) + + out := make(map[string]any) + + switch mediaType { + case "application/x-www-form-urlencoded": + args := c.PostArgs() + args.VisitAll(func(k, v []byte) { + key := string(k) + // 同名 key 多次出现时,转成数组 + if old, exists := out[key]; exists { + switch x := old.(type) { + case string: + out[key] = []string{x, string(v)} + case []string: + out[key] = append(x, string(v)) + default: + out[key] = string(v) + } + } else { + out[key] = string(v) + } + }) + b, e := desensitization(out) + return b, true, e + + case "multipart/form-data": + form, e := c.MultipartForm() + if e != nil { + return nil, false, e + } + + // 普通字段 + for k, vv := range form.Value { + if len(vv) == 1 { + out[k] = vv[0] + } else { + out[k] = vv + } + } + + // 文件字段:只做占位,不读取任何文件信息 + for k, files := range form.File { + if len(files) <= 1 { + out[k] = "[file]" + } else { + arr := make([]string, 0, len(files)) + for range files { + arr = append(arr, "[file]") + } + out[k] = arr + } + } + + b, e2 := desensitization(out) + return b, true, e2 + + default: + // 不是表单就不处理 + return nil, false, nil + } +} + +// ResponseBodySnippet :获取响应内容 +func ResponseBodySnippet(c *app.RequestContext, maxBytes int) (body string, ok bool, truncated bool) { + b := c.Response.Body() + if len(b) == 0 { + return "", false, false + } + + // 只采集“看起来像文本/JSON”的响应,其他(图片/zip等)直接跳过 + ct := strings.ToLower(string(c.Response.Header.ContentType())) + if ct != "" && !(strings.HasPrefix(ct, "text/") || + strings.Contains(ct, "application/json") || + strings.Contains(ct, "application/xml") || + strings.Contains(ct, "application/javascript")) { + return "", false, false + } + + if maxBytes > 0 && len(b) > maxBytes { + b = b[:maxBytes] + truncated = true + } + + body = string(b) + return body, true, truncated +} diff --git a/utils/httplog/uploader.go b/utils/httplog/uploader.go new file mode 100644 index 0000000..9402f91 --- /dev/null +++ b/utils/httplog/uploader.go @@ -0,0 +1,155 @@ +package httplog + +import ( + "apiServer_service/utils/logger" + "apiServer_service/utils/redis_tools" + "bytes" + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" + + "github.com/go-redis/redis/v8" +) + +type AccessEventData struct { + Timestamp string `json:"@timestamp"` + TsMs int64 `json:"ts_ms"` +} + +var popBatchLua = redis.NewScript(` +local key = KEYS[1] +local n = tonumber(ARGV[1]) +local res = redis.call("LRANGE", key, 0, n-1) +if (#res > 0) then + redis.call("LTRIM", key, n, -1) +end +return res +`) + +func getenv(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func getenvInt(key string, def int) int { + v := os.Getenv(key) + if v == "" { + return def + } + i, err := strconv.Atoi(v) + if err != nil { + return def + } + return i +} + +func Updater(redisKey, esIndexPrefix string) { + if redisKey == "" { + redisKey = getenv("ES_REDIS_KEY", "access_log") + } + + esBulkURL := getenv("ES_BULK_URL", "https://elasticsearch.hostidc.net/_bulk") + if esIndexPrefix == "" { + esIndexPrefix = getenv("ES_INDEX_PREFIX", "access") + } + esUser := getenv("ES_USERNAME", "") + esPass := getenv("ES_PASSWORD", "") + + batchSize := getenvInt("ES_BATCH_SIZE", 1000) + pollIntervalMs := getenvInt("ES_POLL_INTERVAL_MS", 200) + httpTimeoutMs := getenvInt("ES_HTTP_TIMEOUT_MS", 5000) + + rdb := redis_tools.ConnectRedis() + + httpClient := &http.Client{Timeout: time.Duration(httpTimeoutMs) * time.Millisecond} + ctx := context.Background() + + for { + items, err := popBatchLua.Run(ctx, rdb, []string{redisKey}, batchSize).StringSlice() + if err != nil { + time.Sleep(time.Second) + continue + } + if len(items) == 0 { + time.Sleep(time.Duration(pollIntervalMs) * time.Millisecond) + continue + } + + body, _ := buildBulkBody(items, esIndexPrefix) + ok, _, err := postBulk(ctx, httpClient, esBulkURL, esUser, esPass, body) + if err != nil || !ok { + logger.Error("ESBulk", "写入失败: ", err) + continue + } + } +} + +func buildBulkBody(items []string, indexPrefix string) ([]byte, int) { + var buf bytes.Buffer + count := 0 + + for _, line := range items { + idx := indexPrefix + "-" + time.Now().Format("2006.01.02") + + var ev AccessEventData + if json.Unmarshal([]byte(line), &ev) == nil { + if ev.TsMs > 0 { + idx = indexPrefix + "-" + time.UnixMilli(ev.TsMs).UTC().Format("2006.01.02") + } else if ev.Timestamp != "" { + if ts, err := time.Parse(time.RFC3339Nano, ev.Timestamp); err == nil { + idx = indexPrefix + "-" + ts.UTC().Format("2006.01.02") + } + } + } + + sum := sha1.Sum([]byte(line)) + docID := hex.EncodeToString(sum[:]) + + meta := fmt.Sprintf(`{"index":{"_index":"%s","_id":"%s"}}`+"\n", idx, docID) + buf.WriteString(meta) + buf.WriteString(line) + buf.WriteByte('\n') + count++ + } + return buf.Bytes(), count +} + +func postBulk(ctx context.Context, client *http.Client, url, user, pass string, body []byte) (bool, []byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return false, nil, err + } + req.Header.Set("Content-Type", "application/x-ndjson") + if user != "" || pass != "" { + req.SetBasicAuth(user, pass) + } + + resp, err := client.Do(req) + if err != nil { + return false, nil, err + } + defer resp.Body.Close() + + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return false, b, fmt.Errorf("bulk http status=%d", resp.StatusCode) + } + + var out struct { + Errors bool `json:"errors"` + } + if json.Unmarshal(b, &out) == nil && out.Errors { + return false, b, fmt.Errorf("bulk response errors=true") + } + + return true, b, nil +} diff --git a/utils/loger/log.go b/utils/loger/log.go deleted file mode 100644 index 62c4f55..0000000 --- a/utils/loger/log.go +++ /dev/null @@ -1,193 +0,0 @@ -package loger - -import ( - "encoding/json" - "fmt" - "github.com/sirupsen/logrus" - "io" - "os" - "reflect" - "runtime" - "strings" - "time" -) - -// ANSI 转义码定义颜色 -const ( - reset = "\033[0m" - red = "\033[31m" - green = "\033[32m" - yellow = "\033[33m" - blue = "\033[34m" - magenta = "\033[35m" - cyan = "\033[36m" - white = "\033[97m" - gray = "\033[90m" - brightYellow = "\033[93m" -) - -// 自定义日志格式化器 -type customFormatter struct{} - -// Format 实现 logrus.Formatter 接口 -func (f *customFormatter) Format(entry *logrus.Entry) ([]byte, error) { - // 根据日志级别选择颜色 - var levelColor string - switch entry.Level { - case logrus.DebugLevel: - levelColor = cyan - case logrus.InfoLevel: - levelColor = green - case logrus.WarnLevel: - levelColor = yellow - case logrus.ErrorLevel: - levelColor = red - case logrus.FatalLevel, logrus.PanicLevel: - levelColor = magenta - default: - levelColor = white - } - - // 获取时间,格式为 yyyy/mm/dd hh:mm:ss.ms - timestamp := fmt.Sprintf("%s%s%s", levelColor, entry.Time.Format("01-02 15:04:05"), reset) - - // 获取文件名和行号 - _, file, line, ok := runtime.Caller(6) // 使用更高的层级 - if !ok { - file = "unknown" - line = 0 - } - if entry.Caller != nil { - // 获取调用者的文件名并只保留文件名部分 - file = file[strings.LastIndex(file, "/")+1:] // 仅保留文件名 - } - - // 获取日志标题 - title, ok := entry.Data["title"].(string) - if !ok { - title = "unknown" - } - - // 格式化日志级别,并为其添加颜色 - level := fmt.Sprintf("%s[%s]%s", levelColor, strings.ToUpper(entry.Level.String()), reset) - title = fmt.Sprintf("%s「%s」%s", levelColor, title, reset) - - // 格式化输出内容 - logMessage := fmt.Sprintf("%s %s:%d: %s >> %s %s\n", - timestamp, - file, - line, - level, - title, - entry.Message, - ) - - return []byte(logMessage), nil -} - -func getLog() *logrus.Logger { - logLevel := os.Getenv("LOG_LEVEL") - logSave := os.Getenv("LOG_SAVE") - logSaveDir := os.Getenv("LOG_SAVE_PATH") - log := logrus.New() - switch logLevel { - case "debug": - log.SetLevel(logrus.DebugLevel) - case "info": - log.SetLevel(logrus.InfoLevel) - case "warn": - log.SetLevel(logrus.WarnLevel) - case "error": - log.SetLevel(logrus.ErrorLevel) - case "fatal": - log.SetLevel(logrus.FatalLevel) - } - // 自定义日志格式 - log.SetFormatter(&customFormatter{}) - // 启用调用信息的追踪,这样可以获取到文件名和行号 - log.SetReportCaller(true) - - if logSave == "true" { - logFileName := time.Now().Format("20060102") + ".log" - logSavePath := logSaveDir + "/" + logFileName - // 判断日志文件是否已存在 - _, err := os.Stat(logSaveDir) - if os.IsNotExist(err) { - // 路径不存在,创建目录 - if err := os.MkdirAll(logSaveDir, 0755); err != nil { - log.Error("日志文件目录创建失败") - return log - } - } - logFile, _ := os.OpenFile(logSavePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - multiWriter := io.MultiWriter(os.Stdout, logFile) - log.SetOutput(multiWriter) - } - return log -} - -func toString(v interface{}) string { - // 使用反射检查类型 - rv := reflect.ValueOf(v) - - // 处理结构体和结构体指针 - if rv.Kind() == reflect.Struct || (rv.Kind() == reflect.Ptr && rv.Elem().Kind() == reflect.Struct) { - jsonData, err := json.Marshal(v) - if err != nil { - return fmt.Sprintf("Error marshalling to JSON: %v", err) - } - return string(jsonData) - } - - // 对于其他类型,使用 fmt.Sprintf - return fmt.Sprintf("%v", v) -} - -// isPrintable 检查字符串是否只包含可打印字符 -func isPrintable(s string) bool { - for _, r := range s { - if r < 32 || r > 126 { - return false - } - } - return true -} - -// joinToString 将多个参数转换为字符串,并拼接在一起。 -func joinToString(parts ...interface{}) string { - var strParts []string - for _, part := range parts { - strParts = append(strParts, toString(part)) - } - return strings.Join(strParts, "") -} - -func Debug(title string, content ...interface{}) { - getLog().WithFields(logrus.Fields{ - "title": title, - }).Debug(joinToString(content)) -} - -func Info(title string, content ...interface{}) { - getLog().WithFields(logrus.Fields{ - "title": title, - }).Info(joinToString(content)) -} - -func Warn(title string, content ...interface{}) { - getLog().WithFields(logrus.Fields{ - "title": title, - }).Warn(joinToString(content)) -} - -func Error(title string, content ...interface{}) { - getLog().WithFields(logrus.Fields{ - "title": title, - }).Error(joinToString(content)) -} - -func Fatal(title string, content ...interface{}) { - getLog().WithFields(logrus.Fields{ - "title": title, - }).Fatal(joinToString(content)) -} diff --git a/utils/logger/log.go b/utils/logger/log.go new file mode 100644 index 0000000..bd0aded --- /dev/null +++ b/utils/logger/log.go @@ -0,0 +1,171 @@ +package logger + +import ( + "encoding/json" + "fmt" + "io" + "os" + "reflect" + "runtime" + "strings" + "sync" + "time" + + "github.com/sirupsen/logrus" +) + +const ( + reset = "\033[0m" + red = "\033[31m" + green = "\033[32m" + yellow = "\033[33m" + blue = "\033[34m" + cyan = "\033[36m" + white = "\033[97m" +) + +type colorFormatter struct{} + +func (f *colorFormatter) Format(entry *logrus.Entry) ([]byte, error) { + var levelColor string + switch entry.Level { + case logrus.DebugLevel: + levelColor = cyan + case logrus.InfoLevel: + levelColor = green + case logrus.WarnLevel: + levelColor = yellow + case logrus.ErrorLevel: + levelColor = red + case logrus.FatalLevel, logrus.PanicLevel: + levelColor = red + default: + levelColor = white + } + + timestamp := fmt.Sprintf("%s%s%s", levelColor, entry.Time.Format("01-02 15:04:05"), reset) + + file := "unknown" + line := 0 + if entry.Caller != nil { + file = entry.Caller.File + if idx := strings.LastIndex(file, "/"); idx >= 0 { + file = file[idx+1:] + } + line = entry.Caller.Line + } + + title, _ := entry.Data["title"].(string) + if title == "" { + title = "-" + } + + level := fmt.Sprintf("%s[%s]%s", levelColor, strings.ToUpper(entry.Level.String()), reset) + titleStr := fmt.Sprintf("%s「%s」%s", levelColor, title, reset) + + msg := fmt.Sprintf("%s %s:%d: %s >> %s %s\n", + timestamp, file, line, level, titleStr, entry.Message, + ) + return []byte(msg), nil +} + +var ( + instance *logrus.Logger + once sync.Once +) + +func GetLogger() *logrus.Logger { + once.Do(func() { + instance = logrus.New() + + switch strings.ToLower(os.Getenv("LOG_LEVEL")) { + case "debug": + instance.SetLevel(logrus.DebugLevel) + case "info": + instance.SetLevel(logrus.InfoLevel) + case "warn": + instance.SetLevel(logrus.WarnLevel) + case "error": + instance.SetLevel(logrus.ErrorLevel) + case "fatal": + instance.SetLevel(logrus.FatalLevel) + default: + instance.SetLevel(logrus.InfoLevel) + } + + instance.SetFormatter(&colorFormatter{}) + instance.SetReportCaller(true) + + if os.Getenv("LOG_SAVE") == "true" { + logDir := os.Getenv("LOG_SAVE_PATH") + if logDir != "" { + if err := os.MkdirAll(logDir, 0755); err != nil { + instance.Errorf("创建日志目录失败: %v", err) + } else { + logFile := logDir + "/" + time.Now().Format("20060102") + ".log" + f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + instance.SetOutput(io.MultiWriter(os.Stdout, f)) + } + } + } + } + }) + return instance +} + +func callerSkip() *runtime.Frame { + pcs := make([]uintptr, 10) + n := runtime.Callers(3, pcs) + frames := runtime.CallersFrames(pcs[:n]) + for { + frame, more := frames.Next() + if !strings.Contains(frame.File, "utils/logger/") { + return &frame + } + if !more { + break + } + } + return nil +} + +func toString(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.Struct || (rv.Kind() == reflect.Ptr && rv.Elem().Kind() == reflect.Struct) { + data, err := json.Marshal(v) + if err != nil { + return fmt.Sprintf("%v", v) + } + return string(data) + } + return fmt.Sprintf("%v", v) +} + +func joinToString(parts ...interface{}) string { + strs := make([]string, 0, len(parts)) + for _, p := range parts { + strs = append(strs, toString(p)) + } + return strings.Join(strs, " ") +} + +func Debug(title string, content ...interface{}) { + GetLogger().WithField("title", title).Debug(joinToString(content...)) +} + +func Info(title string, content ...interface{}) { + GetLogger().WithField("title", title).Info(joinToString(content...)) +} + +func Warn(title string, content ...interface{}) { + GetLogger().WithField("title", title).Warn(joinToString(content...)) +} + +func Error(title string, content ...interface{}) { + GetLogger().WithField("title", title).Error(joinToString(content...)) +} + +func Fatal(title string, content ...interface{}) { + GetLogger().WithField("title", title).Fatal(joinToString(content...)) +} diff --git a/utils/nacos/baseNacos.go b/utils/nacos/baseNacos.go index 6f67702..f71cbf5 100644 --- a/utils/nacos/baseNacos.go +++ b/utils/nacos/baseNacos.go @@ -1,66 +1,87 @@ package nacos import ( - "github.com/nacos-group/nacos-sdk-go/v2/common/constant" + "errors" "net" "os" "strconv" "strings" + "sync" + + "github.com/nacos-group/nacos-sdk-go/v2/common/constant" ) var ( - cc *constant.ClientConfig - sc []constant.ServerConfig + cc *constant.ClientConfig + sc []constant.ServerConfig + initMu sync.Once ) func GetIP(domain string) string { - // 确保域名不为空 if domain == "" { return "" } - - // 使用 net.LookupIP 查找域名的 IP 地址 ips, err := net.LookupIP(domain) - if err != nil { + if err != nil || len(ips) == 0 { return "" } - - // 检查是否找到了 IP - if len(ips) == 0 { - return "" - } - - // 返回第一个 IPv4 地址(如果有) for _, ip := range ips { if ipv4 := ip.To4(); ipv4 != nil { return ipv4.String() } } - - // 如果没有 IPv4,返回第一个 IP(可能是 IPv6) return ips[0].String() } -func InitNacosRegistryConfig() { - if cc != nil && sc != nil { - return - } - nacosHosts := strings.Split(os.Getenv("NACOS_HOSTS"), ",") - nacosPort, _ := strconv.Atoi(os.Getenv("NACOS_PORT")) - for _, host := range nacosHosts { - serverConfig := constant.NewServerConfig(GetIP(host), uint64(nacosPort)) - sc = append(sc, *serverConfig) - } +func InitNacosRegistryConfig() error { + var initErr error + initMu.Do(func() { + hostsStr := os.Getenv("NACOS_HOSTS") + portStr := os.Getenv("NACOS_PORT") + if hostsStr == "" || portStr == "" { + initErr = errors.New("NACOS_HOSTS 和 NACOS_PORT 必须配置") + return + } - LogDir := os.Getenv("LOG_SAVE_PATH") - cc = &constant.ClientConfig{ - NamespaceId: os.Getenv("NACOS_NAMESPACE"), - TimeoutMs: 5000, - NotLoadCacheAtStart: true, - LogDir: LogDir, - //CacheDir: "/tmp/nacos/cache", - LogLevel: "debug", - Username: os.Getenv("NACOS_USER"), - Password: os.Getenv("NACOS_PASSWORD"), - } + nacosPort, err := strconv.Atoi(portStr) + if err != nil { + initErr = errors.New("NACOS_PORT 格式错误") + return + } + + nacosHosts := strings.Split(hostsStr, ",") + for _, host := range nacosHosts { + host = strings.TrimSpace(host) + if host == "" { + continue + } + ip := GetIP(host) + if ip == "" { + ip = host + } + serverConfig := constant.NewServerConfig(ip, uint64(nacosPort)) + sc = append(sc, *serverConfig) + } + + if len(sc) == 0 { + initErr = errors.New("无有效的 Nacos 服务器地址") + return + } + + logDir := os.Getenv("LOG_SAVE_PATH") + if logDir == "" { + logDir = "/tmp/nacos/log" + } + + cc = &constant.ClientConfig{ + NamespaceId: os.Getenv("NACOS_NAMESPACE"), + TimeoutMs: 5000, + NotLoadCacheAtStart: true, + LogDir: logDir, + LogLevel: "warn", + Username: os.Getenv("NACOS_USER"), + Password: os.Getenv("NACOS_PASSWORD"), + } + }) + return initErr } diff --git a/utils/nacos/configuration.go b/utils/nacos/configuration.go index 5b736c8..61750f6 100644 --- a/utils/nacos/configuration.go +++ b/utils/nacos/configuration.go @@ -1,14 +1,13 @@ package nacos import ( - "apiServer_service/utils/loger" + "apiServer_service/utils/logger" "github.com/nacos-group/nacos-sdk-go/v2/clients" "github.com/nacos-group/nacos-sdk-go/v2/clients/config_client" "github.com/nacos-group/nacos-sdk-go/v2/vo" ) -// NewNacosConfigClient 创建一个 Nacos 配置服务 -func NewNacosConfigClient() (*config_client.IConfigClient, error) { +func NewNacosConfigClient() (config_client.IConfigClient, error) { InitNacosRegistryConfig() cli, err := clients.NewConfigClient( vo.NacosClientParam{ @@ -19,39 +18,34 @@ func NewNacosConfigClient() (*config_client.IConfigClient, error) { if err != nil { return nil, err } - return &cli, nil + return cli, nil } -// AddConfig 新增配置 func AddConfig(dataId, group, content string) error { client, err := NewNacosConfigClient() if err != nil { return err } - _, err = (*client).PublishConfig(vo.ConfigParam{ + _, err = client.PublishConfig(vo.ConfigParam{ DataId: dataId, Group: group, Content: content, }) - if err != nil { - return err - } - return nil + return err } -// GetConfig 获取配置 func GetConfig(dataId, group string) string { client, err := NewNacosConfigClient() if err != nil { - loger.Error("获取配置客户端失败", err) + logger.Error("获取配置客户端失败", err) return "" } - content, err := (*client).GetConfig(vo.ConfigParam{ + content, err := client.GetConfig(vo.ConfigParam{ DataId: dataId, Group: group, }) if err != nil { - loger.Error("获取配置失败", err) + logger.Error("获取配置失败", err) return "" } return content diff --git a/utils/nacos/service.go b/utils/nacos/service.go index 1b7ca43..0822e49 100644 --- a/utils/nacos/service.go +++ b/utils/nacos/service.go @@ -1,29 +1,39 @@ package nacos import ( + "errors" + "os" + "strconv" + "sync" + "github.com/nacos-group/nacos-sdk-go/v2/clients" "github.com/nacos-group/nacos-sdk-go/v2/clients/naming_client" "github.com/nacos-group/nacos-sdk-go/v2/model" "github.com/nacos-group/nacos-sdk-go/v2/vo" - "os" - "strconv" ) var ( - cli naming_client.INamingClient - groupName string - ServerUriCache = make(map[string]model.Instance) + namingClient naming_client.INamingClient + namingMu sync.Mutex + groupName string + serviceCache sync.Map ) -// NewNacosRegistry 创建一个nacos注册中心 -func NewNacosRegistry() (*naming_client.INamingClient, error) { - InitNacosRegistryConfig() - if cli != nil { - return &cli, nil +func NewNacosRegistry() (naming_client.INamingClient, error) { + namingMu.Lock() + defer namingMu.Unlock() + + if namingClient != nil { + return namingClient, nil } + + if err := InitNacosRegistryConfig(); err != nil { + return nil, err + } + groupName = os.Getenv("NACOS_GROUP_NAME") var err error - cli, err = clients.NewNamingClient( + namingClient, err = clients.NewNamingClient( vo.NacosClientParam{ ClientConfig: cc, ServerConfigs: sc, @@ -32,17 +42,21 @@ func NewNacosRegistry() (*naming_client.INamingClient, error) { if err != nil { return nil, err } - return &cli, nil + return namingClient, nil } -// RegisterService 注册当前服务到nacos中 func RegisterService() error { client, err := NewNacosRegistry() if err != nil { return err } + serviceName := os.Getenv("NACOS_SERVICE_NAME") host := os.Getenv("NACOS_SERVICE_HOST") + if serviceName == "" || host == "" { + return errors.New("NACOS_SERVICE_NAME 和 NACOS_SERVICE_HOST 必须配置") + } + port, err := strconv.Atoi(os.Getenv("NACOS_SERVICE_PORT")) if err != nil { port = 8848 @@ -51,7 +65,8 @@ func RegisterService() error { if err != nil { weight = 10 } - _, err = (*client).RegisterInstance(vo.RegisterInstanceParam{ + + _, err = client.RegisterInstance(vo.RegisterInstanceParam{ Ip: host, Port: uint64(port), ServiceName: serviceName, @@ -61,43 +76,50 @@ func RegisterService() error { Ephemeral: false, GroupName: groupName, }) - if err != nil { - return err - } - return nil + return err } -// DiscoverServiceList 发现服务列表 func DiscoverServiceList(serviceName string) ([]model.Instance, error) { client, err := NewNacosRegistry() if err != nil { return nil, err } - instances, err := (*client).SelectInstances(vo.SelectInstancesParam{ + return client.SelectInstances(vo.SelectInstancesParam{ ServiceName: serviceName, HealthyOnly: false, GroupName: groupName, }) - if err != nil { - return nil, err - } - return instances, nil } -// DiscoverService 发现一个服务 func DiscoverService(serviceName string) (model.Instance, error) { - ServiceCache := ServerUriCache[serviceName] + if cached, ok := serviceCache.Load(serviceName); ok { + cachedInstance := cached.(model.Instance) + client, err := NewNacosRegistry() + if err != nil { + return cachedInstance, err + } + instance, err := client.SelectOneHealthyInstance(vo.SelectOneHealthInstanceParam{ + ServiceName: serviceName, + GroupName: groupName, + }) + if err != nil { + return cachedInstance, err + } + serviceCache.Store(serviceName, *instance) + return *instance, nil + } + client, err := NewNacosRegistry() if err != nil { - return ServiceCache, err + return model.Instance{}, err } - instances, err := (*client).SelectOneHealthyInstance(vo.SelectOneHealthInstanceParam{ + instance, err := client.SelectOneHealthyInstance(vo.SelectOneHealthInstanceParam{ ServiceName: serviceName, GroupName: groupName, }) if err != nil { - return ServiceCache, err + return model.Instance{}, err } - ServerUriCache[serviceName] = *instances - return *instances, nil + serviceCache.Store(serviceName, *instance) + return *instance, nil } diff --git a/utils/redis_tools/redis_tools.go b/utils/redis_tools/redis_tools.go new file mode 100644 index 0000000..f9b8f0e --- /dev/null +++ b/utils/redis_tools/redis_tools.go @@ -0,0 +1,95 @@ +package redis_tools + +import ( + "apiServer_service/utils/logger" + "context" + "os" + "sync" + "time" + + "github.com/go-redis/redis/v8" +) + +var ( + client *redis.Client + once sync.Once +) + +func ConnectRedis() *redis.Client { + once.Do(func() { + addr := os.Getenv("REDIS_HOST") + if addr == "" { + addr = "127.0.0.1:6379" + } + password := os.Getenv("REDIS_PASSWORD") + + client = redis.NewClient(&redis.Options{ + Addr: addr, + Password: password, + DB: 0, + PoolSize: 20, + MinIdleConns: 5, + DialTimeout: 5 * time.Second, + ReadTimeout: 3 * time.Second, + WriteTimeout: 3 * time.Second, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := client.Ping(ctx).Err(); err != nil { + logger.Warn("Redis", "连接失败: ", err) + return + } + logger.Info("Redis", "连接成功") + }) + return client +} + +func GetClient() *redis.Client { + return ConnectRedis() +} + +func Close() error { + if client != nil { + return client.Close() + } + return nil +} + +// ---- List 操作 ---- + +func AddToList(key string, value interface{}) error { + return GetClient().LPush(context.Background(), key, value).Err() +} + +func GetListRange(key string, start, stop int64) ([]string, error) { + return GetClient().LRange(context.Background(), key, start, stop).Result() +} + +func GetAllFromList(key string) ([]string, error) { + return GetListRange(key, 0, -1) +} + +func RemoveFromList(key string, value interface{}) error { + return GetClient().LRem(context.Background(), key, 0, value).Err() +} + +// ---- KV 缓存操作 ---- + +func SetCache(key string, value interface{}, expiration time.Duration) error { + return GetClient().Set(context.Background(), key, value, expiration).Err() +} + +func GetCache(key string) (string, error) { + return GetClient().Get(context.Background(), key).Result() +} + +func Exists(key string) bool { + n, err := GetClient().Exists(context.Background(), key).Result() + return err == nil && n > 0 +} + +func Del(keys ...string) error { + return GetClient().Del(context.Background(), keys...).Err() +} diff --git a/utils/request/request.go b/utils/request/request.go index d937b60..a42322a 100644 --- a/utils/request/request.go +++ b/utils/request/request.go @@ -1,16 +1,14 @@ package request import ( - "apiServer_service/utils/loger" + "apiServer_service/utils/logger" "fmt" "github.com/cloudwego/hertz/pkg/app" ) -// BindRequestStruct 结构体参数绑定 -func BindRequestStruct(c *app.RequestContext, request interface{}) error { - err := c.BindAndValidate(request) - if err != nil { - loger.Debug("BindRequestStruct", fmt.Sprintf("参数错误: %v", err)) +func BindRequestStruct(c *app.RequestContext, req interface{}) error { + if err := c.BindAndValidate(req); err != nil { + logger.Debug("BindRequestStruct", fmt.Sprintf("参数错误: %v", err)) BadRequest(c, "参数错误") return err } diff --git a/utils/server_cli/serverCli.go b/utils/server_cli/serverCli.go index 300aa7c..8b1b2be 100644 --- a/utils/server_cli/serverCli.go +++ b/utils/server_cli/serverCli.go @@ -2,21 +2,29 @@ package server_cli import ( "apiServer_service/proto" - "apiServer_service/utils/loger" + "apiServer_service/utils/logger" "apiServer_service/utils/nacos" "context" "errors" - "google.golang.org/grpc" - "google.golang.org/grpc/metadata" "os" "strconv" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" +) + +var ( + grpcConn *grpc.ClientConn + grpcClient proto.ServerVisitServiceClient + connMu sync.Mutex ) -// 获取 apiServer 的地址 func getApiServer() string { service, err := nacos.DiscoverService("apiServer") if err != nil { - loger.Error("获取服务器地址失败", err) + logger.Error("获取服务器地址失败", err) if service.Ip != "" { return service.Ip + ":" + strconv.Itoa(int(service.Port)) } @@ -25,42 +33,56 @@ func getApiServer() string { return service.Ip + ":" + strconv.Itoa(int(service.Port)) } -// 获取 grpc 客户端 -func getGrpcClient() (*proto.ServerVisitServiceClient, error) { +func getGrpcClient() (proto.ServerVisitServiceClient, error) { + connMu.Lock() + defer connMu.Unlock() + + if grpcClient != nil && grpcConn != nil { + return grpcClient, nil + } + serverUri := getApiServer() if serverUri == "" { - loger.Error("获取服务器地址失败") - return nil, errors.New("getApiServer error") + return nil, errors.New("无法获取 apiServer 地址") } - conn, err := grpc.NewClient(serverUri, grpc.WithInsecure()) + + conn, err := grpc.NewClient(serverUri, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { - loger.Error("getGrpcClient error", err) + logger.Error("gRPC 连接失败", err) return nil, err } - c := proto.NewServerVisitServiceClient(conn) - return &c, nil + grpcConn = conn + grpcClient = proto.NewServerVisitServiceClient(conn) + return grpcClient, nil } -// getContext 创建带有 token 的上下文 -func getContext() context.Context { // 创建带有 token 的上下文 +func getContext() context.Context { token := os.Getenv("GRPC_TOKEN") - md := metadata.Pairs("authorization", "Bearer "+token) // 设置 authorization 头 - ctx := metadata.NewOutgoingContext(context.Background(), md) - return ctx + md := metadata.Pairs("authorization", "Bearer "+token) + return metadata.NewOutgoingContext(context.Background(), md) } -// ReportVisit 演示方法 向服务器上报访问记录 -func ReportVisit(token, note, VisitIp, OS string, point, UserId int) error { +func ReportVisit(token, note, visitIP, osName string, point, userID int) error { client, err := getGrpcClient() if err != nil { return err } - recode, err := (*client).AddServerVisitRecode(getContext(), &proto.ServerVisitRequest{}) + record, err := client.AddServerVisitRecode(getContext(), &proto.ServerVisitRequest{}) if err != nil { - loger.Error("ReportVisit error", err) + logger.Error("ReportVisit error", err) return err } - loger.Debug("ReportVisit", recode) + logger.Debug("ReportVisit", record) return nil } + +func CloseGrpcConn() { + connMu.Lock() + defer connMu.Unlock() + if grpcConn != nil { + grpcConn.Close() + grpcConn = nil + grpcClient = nil + } +}