nginxのメトリクスとログをモニタリングをする

メトリクスデータは以下のように流す nginx -> vector -> kafka -> opensearch/influxDB という感じで、nginxから出た生ログをvectorでとり、kafkaに送信、kafkaから、opensearchとinfluxDBがそれぞれとってくる、という流れにするのが良さそう で、nginxがログを吐き出し/var/log/nginx/access.logとvectorを同じネームスペースで扱いたいので、この二つは物理マシン上にインストールするという流れにしたいと思う。

環境構成

以下の3つのマシンを使う

1. delta (100.64.1.48,192.168.3.1) : 192.168.3.1/24のルータ,kafka, kafka-ui, opensearch
2. master (192.168.3.8) : プロキシサーバ（nginx）、vector
3. gamma/zeta/　: オリジン

kafkaを導入するdocker composeの設定

services:
  kafka-broker:
    image: apache/kafka:3.7.0
    container_name: kafka-broker
    ports:
      - "${KAFKA_BROKER_LOCAL_PORT}:9092"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT_HOST://localhost:${KAFKA_BROKER_LOCAL_PORT},PLAINTEXT://kafka-broker:${KAFKA_BROKER_PUBLIC_PORT}"
      KAFKA_PROCESS_ROLES: "broker,controller"
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka-broker:${KAFKA_BROKER_CONTROLLER_PORT}"
      KAFKA_LISTENERS: "CONTROLLER://:${KAFKA_BROKER_CONTROLLER_PORT},PLAINTEXT_HOST://:${KAFKA_BROKER_LOCAL_PORT},PLAINTEXT://:${KAFKA_BROKER_PUBLIC_PORT}"
      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_LOG_DIRS: "/tmp/kraft-combined-logs"

  kafka-ui:
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:v0.7.2
    ports:
      - "${KAFKA_UI_PORT}:8080"
    depends_on:
      - kafka-broker
    restart: always
    environment:
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-broker:${KAFKA_BROKER_PUBLIC_PORT}

  init-kafka:
    # kafka-topics コマンドを使いたいので confluenticsのコンテナを利用
    image: confluentinc/cp-kafka:7.6.1
    container_name: init-kafka
    depends_on:
      - kafka-broker
    entrypoint: ["/bin/sh", "-c"]
    command: |
      "
      # blocks until kafka is reachable
      kafka-topics --bootstrap-server kafka-broker:${KAFKA_BROKER_PUBLIC_PORT} --list

      echo -e 'Creating topics'
      kafka-topics --bootstrap-server kafka-broker:${KAFKA_BROKER_PUBLIC_PORT} --create --if-not-exists --topic nginx-log --replication-factor 1 --partitions 1
      
      echo -e 'Successfully created :'
      kafka-topics --bootstrap-server kafka-broker:${KAFKA_BROKER_PUBLIC_PORT} --list
      "

opensearchを導入する設定

version: '3'
services:
  opensearch-node1: # This is also the hostname of the container within the Docker network (i.e. https://opensearch-node1/)
    image: opensearchproject/opensearch:latest # Specifying the latest available image - modify if you want a specific version
    container_name: opensearch-node1
    environment:
      - cluster.name=opensearch-cluster # Name the cluster
      - node.name=opensearch-node1 # Name the node that will run in this container
      - discovery.seed_hosts=opensearch-node1,opensearch-node2 # Nodes to look for when discovering the cluster
      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Nodes eligible to serve as cluster manager
      - bootstrap.memory_lock=true # Disable JVM heap memory swapping
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM
      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}    # Sets the demo admin user password when using demo configuration, required for OpenSearch 2.12 and later
    ulimits:
      memlock:
        soft: -1 # Set memlock to unlimited (no soft or hard limit)
        hard: -1
      nofile:
        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
        hard: 65536
    volumes:
      - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
    ports:
      - 9200:9200 # REST API
      - 9600:9600 # Performance Analyzer
    networks:
      - opensearch-net # All of the containers will join the same Docker bridge network
  opensearch-node2:
    image: opensearchproject/opensearch:latest # This should be the same image used for opensearch-node1 to avoid issues
    container_name: opensearch-node2
    environment:
      - cluster.name=opensearch-cluster
      - node.name=opensearch-node2
      - discovery.seed_hosts=opensearch-node1,opensearch-node2
      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2
      - bootstrap.memory_lock=true
      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    volumes:
      - opensearch-data2:/usr/share/opensearch/data
    networks:
      - opensearch-net
  opensearch-dashboards:
    image: opensearchproject/opensearch-dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes
    container_name: opensearch-dashboards
    ports:
      - 5601:5601 # Map host port 5601 to container port 5601
    expose:
      - "5601" # Expose port 5601 for web access to OpenSearch Dashboards
    environment:
      OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query
    networks:
      - opensearch-net

volumes:
  opensearch-data1:
  opensearch-data2:

networks:
  opensearch-net:

nginxを導入する設定

これは普通に

sudo apt install nginx

で一発です。

vectorを導入するプロセス

これもそんなに難しくないでしょう。