March 8, 2021

Kafka server setup

Script to start a kafka server

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#!/bin/sh

setProperty(){
    awk -v pat="^$1=" -v value="$1=$2" '{ if ($0 ~ pat) print value; else print $0; }' $3 > $3.tmp
    mv $3.tmp $3
}

downloadAndPrepare() {
    DIR="kafka/"
    if [[ -d "$DIR" ]]; then
        cd kafka
        rm -rf nohup.out
    else
        wget https://www-us.apache.org/dist/kafka/2.3.0/kafka_2.12-2.3.0.tgz
        tar -xzf kafka_2.12-2.3.0.tgz
        mv kafka_2.12-2.3.0 kafka
        cd kafka
        rm -rf kafka_2.12-2.3.0.tgz
    fi
}

stopServer() {
    PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}')

    if [[ -z "$PIDS" ]]; then
        echo "No kafka server to stop"
    else
        for pid in $PIDS; do kill -9 $pid; done
    fi

    PIDS=$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}')

    if [[ -z "$PIDS" ]]; then
        echo "No zookeeper server to stop"
    else
        for pid in $PIDS; do kill -9 $pid; done
    fi

}

startServer() {
    nohup bin/zookeeper-server-start.sh config/zookeeper.properties &
    nohup bin/kafka-server-start.sh config/server.properties &
}

helpFunction()
{
    echo ""
    echo "Usage: $0 -n number"
    echo -e "\t-n Number of kafka server to launch"
    exit 1
}

while getopts "n:" opt
do
    case "$opt" in
        n ) number="$OPTARG" ;;
        ? ) helpFunction ;;
    esac
done

if [[ -z "$number" ]] || ! [[ "$number" =~ ^[0-9]+$ ]]
then
    helpFunction
fi

downloadAndPrepare
stopServer
startServer

counter=1
while [[ ${counter} -lt ${number} ]]
do
    cp config/server.properties config/"server-$counter.properties"

    setProperty "broker.id" "$counter" config/"server-$counter.properties"
    setProperty "log.dirs" "/tmp/kafka-logs-$counter" config/"server-$counter.properties"
    echo "listeners=PLAINTEXT://:$((counter+9092))" >> config/"server-$counter.properties"

    nohup bin/kafka-server-start.sh config/"server-$counter.properties" &

    counter=$(( counter+1 ))
done

About this Post

This post is written by Mahfuzur Rahman, licensed under CC BY-NC 4.0.

#kafka