Apakah benar-benar tidak mungkin melakukannya tanpa kafoka dan kelinci saat Anda menerima 10.000 peristiwa per detik?

Saya pernah menyelenggarakan webinar tentang cara menerima 10.000 acara per detik. Dia menunjukkan gambar ini, penonton melihat lapisan ungu, dan itu dimulai: "Teman-teman, mengapa kita membutuhkan semua kafka dan kelinci ini, apakah kita benar-benar tidak bisa hidup tanpanya?" Kami menjawab: "Mengapa, mengapa, melewati jaminan sosial!" 

Sangat lucu, tapi biar saya jelaskan.

---

.

,

, , . Kafka, Rabbit . , . 

: 10 , — , — 50 . , . 2 , 10 . , . — .

— -, . 

, . . : , . http- — 5, 10 — , . 

. « », : « , , ». «» , .

— — , . 

, , . , , , e-mail . , , , .

— — -, .

, . ? , . , . - , — (, ). , .

---

, .

-. . .

resource "yandex_compute_instance_group" "events-api-ig" {
  name               = "events-api-ig"
  service_account_id = yandex_iam_service_account.instances.id
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

. CPU, , ..

instance_template {
    platform_id = "standard-v2"
    resources {
      memory = 2
      cores  = 2
    }
    boot_disk {
      mode = "READ_WRITE"
      initialize_params {
        image_id = data.yandex_compute_image.container-optimized-image.id
        size = 10
      }
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

, .

}
    network_interface {
      network_id = yandex_vpc_network.internal.id
      subnet_ids = [yandex_vpc_subnet.internal-a.id, yandex_vpc_subnet.internal-b.id, yandex_vpc_subnet.internal-c.id]
      nat = true
    }
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

— scale_policy.

fixed scale A, B, C.

scale_policy {
    fixed_scale {
      size = 3
    }
  }

  allocation_policy {
    zones = ["ru-central1-a", "ru-central1-b", "ru-central1-c"]
  }
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

auto_scale — .

scale_policy {
auto_scale {
    initial_size = 3
    measurment_duration = 60
    cpu_utilization_target = 60
    min_zone_size = 1
    max_size = 6
    warmup_duration = 60
    stabilization_duration = 180
}
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

, , — cpu utilization target. , . .

, .

- load-. , 84.201.147.84 80, - — 8080. 

, Yandex.Tank . 20 5 . 

---

, .

(A, B C), , . .

, . , .

. , , ( ) 23 , 12,8 . - . , .

— .

, , CPU . : overload.yandex.net/256194.

---

golang. .

package main

import (
    "encoding/json"
    "flag"
    "io"
    "io/ioutil"
    "log"
    "net/http"
    "strings"

)
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

github.com/Shopify/sarama — .

github.com/prometheus/client_golang/prometheus, API Metrics.

github.com/streadway/amqp rabbitmq.

, .

var (
    // Config options
    addr     = flag.String("addr", ":8080", "TCP address to listen to")
    kafka    = flag.String("kafka", "127.0.0.1:9092", "Kafka endpoints")
    enableKafka    = flag.Bool("enable-kafka", false, "Enable Kafka or not")
amqp    = flag.String("amqp", "amqp://guest:guest@127.0.0.1:5672/", "AMQP URI")
enableAmqp    = flag.Bool("enable-amqp", false, "Enable AMQP or not")
sqsUri    = flag.String("sqs-uri", "", "SQS URI")
sqsId    = flag.String("sqs-id", "", SQS Access id")
sqsSecret    = flag.String("sqs-secret", "", "SQS Secret key")
enableSqs    = flag.Bool("enable-sqs", false, "Enable SQS or not")

    

    // Declaring prometheus metrics
    apiDurations = prometheus.NewSummary(
        prometheus.SummaryOpts{
            Name:       "api_durations_seconds",
            Help:       "API duration seconds",
            Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
        },
    )
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

— ().

— .

.

— .

— amqp .

— sqs ..

. prometheus .

main , Load.

sqs, ..

http :

/status okey, load-, .

/post/kafka, . /post/amqp /post/sqs.

— , . , .

- . , SSD ( , ), zookeeper. 200 ! — « », . 120 .

, — CPU, , - . , .

: topic, , . topic 50 partitions. .

, topic load 3 . Partition 1 Kafka 1, — 2, — 3. . , , , . .

50 , 50 1 — 50 . .

zookeeper. 3 . , , 2 . 2 2 . , : «, », 2 .

Terraform

terraform-, kafka.tf .

3 : resource “yandex compute instance” “zookeeper count = 3”.

“zookeeper_deploy”, . , , . . ansible .

, , .

RabbitMQ 

, , . , ! , .

— exchanges , delayed messages, deadletter . . , . — . , , . — .

. , , .

— RabbitMQ. — , , .. — kafka. .

: .