nginx-ingress-controller日志持久化方案的解决

---
apiVersion: v1
kind: Service
metadata:
 name: ingress-nginx
 namespace: kube-system
spec:
 type: ClusterIP
 ports:
 - name: http
  port: 80
  targetPort: 80
  protocol: TCP
 - name: https
  port: 443
  targetPort: 443
  protocol: TCP
 selector:
  app: ingress-nginx
---
apiVersion: v1
kind: Service
metadata:
 name: default-http-backend
 namespace: kube-system
 labels:
  app: default-http-backend
spec:
 ports:
 - port: 80
  targetPort: 8080
 selector:
  app: default-http-backend
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
 name: default
 namespace: kube-system
spec:
 backend:
  serviceName: default-http-backend
  servicePort: 80
---
kind: ConfigMap
apiVersion: v1
metadata:
 name: nginx-configuration
 namespace: kube-system
 labels:
  app: ingress-nginx
data:
 use-forwarded-headers: "true"
 # 此处配置nginx日志的重定向目标
 access-log-path: /var/log/nginx_ingress_controller/access.log
 error-log-path: /var/log/nginx_ingress_controller/error.log

---

# 创建一个configmap，配置nginx日志的轮转策略，对应的是nginx日志在容器内的日志文件
apiVersion: v1
data:
 nginx.log: |
  {{ user_nginx_log.host_path }}/access.log {
    rotate {{ user_nginx_log.rotate_count }}
    daily
    maxsize {{ user_nginx_log.rotate_size }}
    minsize 10M
    copytruncate
    missingok
    create 0644 root root
  }
  {{ user_nginx_log.host_path }}/error.log {
    rotate {{ user_nginx_log.rotate_count }}
    daily
    maxsize {{ user_nginx_log.rotate_size }}
    minsize 10M
    copytruncate
    missingok
    create 0644 root root
  }
kind: ConfigMap
metadata:
 name: nginx-ingress-logrotate
 namespace: kube-system
---

kind: ConfigMap
apiVersion: v1
metadata:
 name: tcp-services
 namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
 name: udp-services
 namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
 name: nginx-ingress-serviceaccount
 namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
 name: nginx-ingress-clusterrole
rules:
 - apiGroups:
   - ""
  resources:
   - configmaps
   - endpoints
   - nodes
   - pods
   - secrets
  verbs:
   - list
   - watch
 - apiGroups:
   - ""
  resources:
   - nodes
  verbs:
   - get
 - apiGroups:
   - ""
  resources:
   - services
  verbs:
   - get
   - list
   - watch
 - apiGroups:
   - "extensions"
  resources:
   - ingresses
  verbs:
   - get
   - list
   - watch
 - apiGroups:
   - ""
  resources:
    - events
  verbs:
    - create
    - patch
 - apiGroups:
   - "extensions"
  resources:
   - ingresses/status
  verbs:
   - update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
 name: nginx-ingress-role
 namespace: kube-system
rules:
 - apiGroups:
   - ""
  resources:
   - configmaps
   - pods
   - secrets
   - namespaces
  verbs:
   - get
 - apiGroups:
   - ""
  resources:
   - configmaps
  resourceNames:
   # Defaults to "
-
"
   # Here: "
-
"
   # This has to be adapted if you change either parameter
   # when launching the nginx-ingress-controller.
   - "ingress-controller-leader-nginx"
  verbs:
   - get
   - update
 - apiGroups:
   - ""
  resources:
   - configmaps
  verbs:
   - create
 - apiGroups:
   - ""
  resources:
   - endpoints
  verbs:
   - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
 name: nginx-ingress-role-nisa-binding
 namespace: kube-system
roleRef:
 apiGroup: rbac.authorization.k8s.io
 kind: Role
 name: nginx-ingress-role
subjects:
 - kind: ServiceAccount
  name: nginx-ingress-serviceaccount
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
 name: nginx-ingress-clusterrole-nisa-binding
roleRef:
 apiGroup: rbac.authorization.k8s.io
 kind: ClusterRole
 name: nginx-ingress-clusterrole
subjects:
 - kind: ServiceAccount
  name: nginx-ingress-serviceaccount
  namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
 name: ingress-nginx
 namespace: kube-system
spec:
 selector:
  matchLabels:
   app: ingress-nginx
 template:
  metadata:
   labels:
    app: ingress-nginx
   annotations:
    prometheus.io/port: '10254'
    prometheus.io/scrape: 'true'
  spec:
   serviceAccountName: nginx-ingress-serviceaccount
   tolerations:
   - key: dedicated
    value: ingress-nginx
    effect: NoSchedule
   affinity:
    nodeAffinity:
     requiredDuringSchedulingIgnoredDuringExecution:
      nodeSelectorTerms:
      - matchExpressions:
       - key: "system/ingress"
        operator: In
        values:
        - "true"
   dnsPolicy: ClusterFirstWithHostNet
   hostNetwork: true
   # 配置initcontainer，确保在nginx-ingress-controller容器启动前将日志目录的权限配置好
   initContainers:
   - name: adddirperm
    image: "{{ image_registry.addr }}/{{ image.adddirperm }}"
    env:
    - name: LOG_DIR
     value: /var/log/nginx_ingress_controller
    - name: USER_ID
      value: "33"
    volumeMounts:
    - name: logdir
     mountPath: /var/log/nginx_ingress_controller
   containers:
   - name: nginx-ingress-controller
    image: "{{ image_registry.addr }}/{{ image.ingress }}"
    imagePullPolicy: IfNotPresent
    args:
    - /nginx-ingress-controller
    - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
    - --configmap=$(POD_NAMESPACE)/nginx-configuration
    - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
    - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
    - --publish-service=$(POD_NAMESPACE)/ingress-nginx
    - --annotations-prefix=nginx.ingress.kubernetes.io
    
    # 设置controller日志的输出路径和方式
    - --log_dir=/var/log/nginx_ingress_controller
    - --logtostderr=false
    securityContext:
     capabilities:
       drop:
       - ALL
       add:
       - NET_BIND_SERVICE
     # www-data -> 33
     runAsUser: 33
    env:
     - name: POD_NAME
      valueFrom:
       fieldRef:
        fieldPath: metadata.name
     - name: POD_NAMESPACE
      valueFrom:
       fieldRef:
        fieldPath: metadata.namespace
    ports:
    - name: http
     containerPort: 80
    - name: https
     containerPort: 443
    resources:
     requests:
      cpu: 100m
      memory: 256Mi
    livenessProbe:
     failureThreshold: 3
     httpGet:
      path: /healthz
      port: 10254
      scheme: HTTP
     initialDelaySeconds: 10
     periodSeconds: 10
     successThreshold: 1
     timeoutSeconds: 1
    readinessProbe:
     failureThreshold: 3
     httpGet:
      path: /healthz
      port: 10254
      scheme: HTTP
     periodSeconds: 10
     successThreshold: 1
     timeoutSeconds: 1
    volumeMounts:
    # 配置挂载容器中控制器组件和nginx的日志输出路径
    - name: logdir
     mountPath: /var/log/nginx_ingress_controller
    # 配置nginx日志的logrotate配置挂载路径
    - name: logrotateconf
     mountPath: /etc/logrotate.d/nginx.log
     subPath: nginx.log
   volumes:
   # 控制器组件和nginx的日志输出路径为宿主机的hostpath
   - name: logdir
    hostPath:
     path: {{ user_nginx_log.host_path }}
     type: ""
   # nginx日志的轮转配置文件来自于configmap
   - name: logrotateconf
    configMap:
     name: nginx-ingress-logrotate
     items:
     - key: nginx.log
      path: nginx.log
---

apiVersion: apps/v1
kind: DaemonSet
metadata:
 name: default-http-backend
 namespace: kube-system
 labels:
  app: default-http-backend
spec:
 selector:
  matchLabels:
   app: default-http-backend
 template:
  metadata:
   labels:
    app: default-http-backend
  spec:
   terminationGracePeriodSeconds: 60
   tolerations:
   - key: dedicated
    value: ingress-nginx
    effect: NoSchedule
   affinity:
    nodeAffinity:
     requiredDuringSchedulingIgnoredDuringExecution:
      nodeSelectorTerms:
      - matchExpressions:
       - key: "system/ingress"
        operator: In
        values:
        - "true"
   containers:
   - name: default-http-backend
    # Any image is permissible as long as:
    # 1. It serves a 404 page at /
    # 2. It serves 200 on a /healthz endpoint
    image: "{{ image_registry.addr }}/{{ image.http_backend }}"
    imagePullPolicy: IfNotPresent
    livenessProbe:
     httpGet:
      path: /healthz
      port: 8080
      scheme: HTTP
     initialDelaySeconds: 30
     timeoutSeconds: 5
    ports:
    - containerPort: 8080
    resources:
     limits:
      cpu: 10m
      memory: 20Mi
     requests:
      cpu: 10m
      memory: 20Mi
---

最后，有的人建议将initcontainer去掉，改为基于原有的nginx-ingress-controller镜像加一层layer，将配置路径权限的脚本放在该层执行。 个人认为这种方法既不美观，也不方便。唯一的好处仅在于deploy yaml仍然简洁（但少不了volumeMount之类的配置）。不过还是看个人使用感受吧~