基于Nginx 搭建缓存服务（nginx+redis+srcache）

随着用户访问请求的压力的剧增，服务器可能处理不过增量太快的请求。单用户请求的响应时间将会变长。如果说有一个什么方案来解决这个问题，那缓存肯定是首选。

不改动代码的情况下使用 nginx 来做缓存处理，两种选择：1，nginx+redis+srcache；2，nginx-memcached+srcache 两种方式都是使用nginx modules模块, 由于memcached有存储大小限制，不得超过1M，我选择了redis服务，还是老样子使用容器，开始编译镜像。官方的 nginx modules 使用说明, srcache-nginx-module 原理说明

版本说明

Ubuntu：18.04

Nginx:1.19.7

Redis:5.0.7

需编译的nginx模块

ngx_http_redis2_module.so
ngx_http_redis_module.so
ngx_http_set_misc_module.so
ngx_http_srcache_filter_module.so
ngx_http_echo_module.so
ndk_http_module.so

需要的配置文件及目录

<!--Dockerfile-->
Dockerfile
<!--docker-compose.yml 启动配置-->
docker-compose.yml
<!--启动脚本-->
update.sh
<!--编译脚本-->
make-image.sh
<!--测试页面-->
./app/index.html
<!--nginix 配置文件-->
./config/nginx-redis-srcache.conf

其他说明

编译过程中遇到许多坑，最终是使用 Ubuntu:18.04 一步步实践过来

1，准备配置文件

1.1 Dockerfile 配置文件

1<!--不需要编译的镜像-->
2docker pull halobug/nginx-redis-srcache:1.19.7

<!--Dockerfile-->

FROM ubuntu:18.04

ARG NGINX_VERSION=1.19.7
# 换源
RUN sed -i "s/archive.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list

RUN apt update && apt-get install -y libpcre3 libpcre3-dev gcc openssl libssl-dev unzip zip make wget

RUN mkdir /nginx-dir && cd /nginx-dir && \
    wget -c http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz -O nginx.tar.gz && mkdir nginx && tar -zxvf nginx.tar.gz -C ./nginx --strip-components=1 && rm nginx.tar.gz && \
    wget -c https://github.com/openresty/redis2-nginx-module/archive/refs/tags/v0.15.tar.gz -O redis2-nginx-module.tar.gz && mkdir redis2-nginx-module && tar -zxvf redis2-nginx-module.tar.gz -C ./redis2-nginx-module --strip-components=1 && rm redis2-nginx-module.tar.gz && \
    wget -c https://people.freebsd.org/~osa/ngx_http_redis-0.3.8.tar.gz -O ngx_http_redis.tar.gz && mkdir ngx_http_redis && tar -zxvf ngx_http_redis.tar.gz -C ./ngx_http_redis  --strip-components=1 && rm ngx_http_redis.tar.gz && \
    wget -c https://github.com/openresty/set-misc-nginx-module/archive/refs/tags/v0.32.tar.gz -O set-misc-nginx-module.tar.gz && mkdir set-misc-nginx-module && tar -zxvf set-misc-nginx-module.tar.gz -C ./set-misc-nginx-module --strip-components=1 && rm set-misc-nginx-module.tar.gz  && \
    wget -c https://github.com/openresty/srcache-nginx-module/archive/refs/tags/v0.32.tar.gz -O srcache-nginx-module.tar.gz && mkdir srcache-nginx-module && tar -zxvf srcache-nginx-module.tar.gz -C ./srcache-nginx-module --strip-components=1 && rm srcache-nginx-module.tar.gz &&\
    wget -c https://github.com/openresty/echo-nginx-module/archive/refs/tags/v0.62.tar.gz -O echo-nginx-module.tar.gz && mkdir echo-nginx-module && tar -zxvf echo-nginx-module.tar.gz -C ./echo-nginx-module --strip-components=1 && rm echo-nginx-module.tar.gz && \
    wget -c https://github.com/vision5/ngx_devel_kit/archive/refs/heads/master.zip -O ngx_devel_kit.zip && mkdir ngx_devel_kit && unzip -d ./ngx_devel_kit ngx_devel_kit.zip && mv ./ngx_devel_kit/ngx_devel_kit-master/* ./ngx_devel_kit/ && rm -r ./ngx_devel_kit/ngx_devel_kit-master && rm -rf  && rm ngx_devel_kit.zip

RUN useradd -M -s /sbin/nologin nginx

RUN apt-get install -y zlib1g-dev

RUN mv /nginx-dir/nginx /usr/src/nginx && cd /usr/src/nginx &&\ 
    ./configure --user=nginx --group=nginx --prefix=/usr --sbin-path=/usr/sbin/nginx \ 
    --conf-path=/etc/nginx/nginx.conf \
    --error-log-path=/var/log/nginx/error.log \
    --http-log-path=/var/log/nginx/access.log \
    --pid-path=/var/run/nginx/nginx.pid \
    --lock-path=/var/lock/nginx.lock \ 
    --add-dynamic-module=/nginx-dir/redis2-nginx-module \
    --add-dynamic-module=/nginx-dir/ngx_http_redis \
    --add-dynamic-module=/nginx-dir/ngx_devel_kit \
    --add-dynamic-module=/nginx-dir/set-misc-nginx-module \
    --add-dynamic-module=/nginx-dir/srcache-nginx-module \
    --add-dynamic-module=/nginx-dir/echo-nginx-module \
    && make && make install

EXPOSE 80

RUN apt clean

RUN rm -rf /usr/src/nginx && rm -rf /nginx-dir

CMD ["nginx", "-g","daemon off;"]

1.2 docker-compose.yml 配置文件

version: "3.6"

services:
  local-halobug:
    image: halobug/nginx-redis-srcache:1.19.7
    restart: always
    expose:
      - 80
    networks:
      - traefik
    depends_on:
      - redis_cache
    volumes:
      # 使用缓存配置
      - ./config/nginx-redis-srcache.conf:/etc/nginx/nginx.conf
      - ./app:/usr/share/nginx/html
      # 不使用缓存测试
      # - ./app:/usr/html
      - ./logs:/var/log/nginx
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=traefik"
      - "traefik.http.routers.local_halobug.entrypoints=https"
      - "traefik.http.routers.local_halobug.rule=Host(`nginx.halobug.cn`)"
      - "traefik.http.routers.local_halobug.tls=true"
      - "traefik.http.services.local_halobug-backend.loadbalancer.server.scheme=http"
      - "traefik.http.services.local_halobug-backend.loadbalancer.server.port=80"
    logging:
      driver: "json-file"
      options:
        max-size: "1m"

  redis_cache:
    container_name: redis_cache
    image: redis:5.0.7-alpine
    restart: always
    expose:
      - 6379
    networks:
      - traefik
networks:
  traefik:
    external: true

1.3 update.sh 配置文件

#!/usr/bin/env bash

# 关闭服务
echo "stop services."
docker-compose down --remove-orphans

# 重启服务
echo 'restart services.'
docker-compose up -d

1.4 make-image.sh 配置文件

#!/usr/bin/env bash

docker build -t halobug/nginx-redis-srcache:1.19.7 .

1.5 ./config/nginx-redis-srcache.conf 配置文件

load_module modules/ndk_http_module.so;
load_module modules/ngx_http_set_misc_module.so;
load_module modules/ngx_http_echo_module.so;
load_module modules/ngx_http_redis_module.so;
load_module modules/ngx_http_redis2_module.so;
load_module modules/ngx_http_srcache_filter_module.so;

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;
events {
    worker_connections  1024;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    upstream redis {
        server redis_cache:6379;
        keepalive 10;
    }
    server {
        listen 80;
        server_name _;
        charset utf-8;
        gzip on;
        location = /redis_get {
            internal;
            set_md5 $redis_key $args;
            redis_pass redis;
        }
        location = /redis_set {
            internal;
            set_unescape_uri $exptime $arg_exptime;
            set_unescape_uri $key $arg_key;
            set_md5 $key;
            redis2_query set $key $echo_request_body;
            redis2_query expire $key $exptime;
            redis2_pass redis;
        }
        root   /usr/share/nginx/html;
        # srcache_store_private on;
        # srcache_methods GET;
        # srcache_response_cache_control off;
        location / {
            # 根据路径设置key
            set $key $uri$args;
            set_escape_uri $escaped_key $key;

            srcache_fetch GET /redis_get $key;
            srcache_store PUT /redis_set key=$escaped_key&exptime=120;
            # set header
            add_header X-Cached-From $srcache_fetch_status;
            set_md5 $md5key $key;
            add_header X-md5-key $md5key;
            add_header X-Cached-Store $srcache_store_status;
            add_header X-Key $key;
            add_header X-Query_String $query_string;
            add_header X-expire $srcache_expire;

        }
    }
    include /etc/nginx/conf.d/*.conf;
}