Nginx2vector2kafka2opensearch
nginxのメトリクスとログをモニタリングをする メトリクスデータは以下のように流す nginx -> vector -> kafka -> opensearch/influxDB という感じで、nginxから出た生ログをvectorでとり、kafkaに送信、kafkaから、opensearchとinfluxDBがそれぞれとってくる、という流れにするのが良さそう で、nginxがログを吐き出し/var/log/nginx/access.logとvectorを同じネームスペースで扱いたいので、この二つは物理マシン上にインストールするという流れにしたいと思う。 環境構成 以下の3つのマシンを使う delta (100.64.1.48,192.168.3.1) : 192.168.3.1/24のルータ,kafka, kafka-ui, opensearch master (192.168.3.8) : プロキシサーバ(nginx)、vector gamma/zeta/ : オリジン kafkaを導入するdocker composeの設定 services: kafka-broker: image: apache/kafka:3.7.0 container_name: kafka-broker ports: - "${KAFKA_BROKER_LOCAL_PORT}:9092" environment: KAFKA_NODE_ID: 1 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT_HOST://localhost:${KAFKA_BROKER_LOCAL_PORT},PLAINTEXT://kafka-broker:${KAFKA_BROKER_PUBLIC_PORT}" KAFKA_PROCESS_ROLES: "broker,controller" KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka-broker:${KAFKA_BROKER_CONTROLLER_PORT}" KAFKA_LISTENERS: "CONTROLLER://:${KAFKA_BROKER_CONTROLLER_PORT},PLAINTEXT_HOST://:${KAFKA_BROKER_LOCAL_PORT},PLAINTEXT://:${KAFKA_BROKER_PUBLIC_PORT}" KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 KAFKA_LOG_DIRS: "/tmp/kraft-combined-logs" kafka-ui: container_name: kafka-ui image: provectuslabs/kafka-ui:v0.7.2 ports: - "${KAFKA_UI_PORT}:8080" depends_on: - kafka-broker restart: always environment: KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-broker:${KAFKA_BROKER_PUBLIC_PORT} init-kafka: # kafka-topics コマンドを使いたいので confluenticsのコンテナを利用 image: confluentinc/cp-kafka:7.6.1 container_name: init-kafka depends_on: - kafka-broker entrypoint: ["/bin/sh", "-c"] command: | " # blocks until kafka is reachable kafka-topics --bootstrap-server kafka-broker:${KAFKA_BROKER_PUBLIC_PORT} --list echo -e 'Creating topics' kafka-topics --bootstrap-server kafka-broker:${KAFKA_BROKER_PUBLIC_PORT} --create --if-not-exists --topic nginx-log --replication-factor 1 --partitions 1 echo -e 'Successfully created :' kafka-topics --bootstrap-server kafka-broker:${KAFKA_BROKER_PUBLIC_PORT} --list " opensearchを導入する設定 version: '3' services: opensearch-node1: # This is also the hostname of the container within the Docker network (i.e. https://opensearch-node1/) image: opensearchproject/opensearch:latest # Specifying the latest available image - modify if you want a specific version container_name: opensearch-node1 environment: - cluster.name=opensearch-cluster # Name the cluster - node.name=opensearch-node1 # Name the node that will run in this container - discovery.seed_hosts=opensearch-node1,opensearch-node2 # Nodes to look for when discovering the cluster - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 # Nodes eligible to serve as cluster manager - bootstrap.memory_lock=true # Disable JVM heap memory swapping - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} # Sets the demo admin user password when using demo configuration, required for OpenSearch 2.12 and later ulimits: memlock: soft: -1 # Set memlock to unlimited (no soft or hard limit) hard: -1 nofile: soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536 hard: 65536 volumes: - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container ports: - 9200:9200 # REST API - 9600:9600 # Performance Analyzer networks: - opensearch-net # All of the containers will join the same Docker bridge network opensearch-node2: image: opensearchproject/opensearch:latest # This should be the same image used for opensearch-node1 to avoid issues container_name: opensearch-node2 environment: - cluster.name=opensearch-cluster - node.name=opensearch-node2 - discovery.seed_hosts=opensearch-node1,opensearch-node2 - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2 - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} ulimits: memlock: soft: -1 hard: -1 nofile: soft: 65536 hard: 65536 volumes: - opensearch-data2:/usr/share/opensearch/data networks: - opensearch-net opensearch-dashboards: image: opensearchproject/opensearch-dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes container_name: opensearch-dashboards ports: - 5601:5601 # Map host port 5601 to container port 5601 expose: - "5601" # Expose port 5601 for web access to OpenSearch Dashboards environment: OPENSEARCH_HOSTS: '["https://opensearch-node1:9200","https://opensearch-node2:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query networks: - opensearch-net volumes: opensearch-data1: opensearch-data2: networks: opensearch-net: nginxを導入する設定 これは普通に ...