I frequently saw connection has TIME_WAIT status, for ex:

1
2
3
4
5
????netstat -ant|awk '/^tcp/ {++S[$NF]} END {for(a in S) print (a,S[a])}'
LISTEN 35
CLOSE_WAIT 22
TIME_WAIT 3
ESTABLISHED 55

how TIME_WAIT happens?

let’s do some experiments.

server.c

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
#include <stdio.h>
#include <stdlib.h> #include <sys/socket.h>
#include <netinet/in.h>
#include <string.h>
#include <unistd.h>

#define TRUE 1
#define FALSE 0

void error_handling(char *message);

int main(int argc, char *argv[]) {
    int serv_sock, clnt_sock;
    char message[30];
    int option, str_len;
    socklen_t optlen, clnt_adr_sz;
    struct sockaddr_in serv_adr, clnt_adr;

    if (argc != 2) {
        printf("Usage : %s <port> \n", argv[0]);
        exit(1);
    }

    serv_sock = socket(PF_INET, SOCK_STREAM, 0);
    if (serv_sock == -1) {
        error_handling("socket() error");
    }
/*
    optlen = sizeof(option);
    option = TRUE;
    state = setsockopt(serv_sock, SOL_SOCKET, SO_REUSEADDR, (void *) &option, optlen);
    if (state) {
        error_handling("setsockopt() error");
    }
*/
    memset(&serv_adr, 0, sizeof(serv_adr));
    serv_adr.sin_family = AF_INET;
    serv_adr.sin_addr.s_addr = htonl(INADDR_ANY);
    serv_adr.sin_port = htons(atoi(argv[1]));

    if (bind(serv_sock, (struct sockaddr *) &serv_adr, sizeof(serv_adr))) {
        error_handling("bind() error");
    }
    if (listen(serv_sock, 5) == -1) {
        error_handling("listen() error");
    }
    clnt_adr_sz = sizeof(clnt_adr);
    clnt_sock = accept(serv_sock, (struct sockaddr *) &clnt_adr, &clnt_adr_sz);
    if (clnt_sock == -1) {
        error_handling("accept() error");
    }

    while ((str_len = read(clnt_sock, message, sizeof(message))) != 0) {
        write(clnt_sock, message, str_len);
        write(1, message, str_len);
    }
    close(clnt_sock);
    close(serv_sock);
    return 0;
}

void error_handling(char *message) {
    fputs(message, stderr);
    fputc('\n', stderr);
    exit(1);
}

in this code, we keep reading message send from client, then write it back to client, at the same time, we will write the message to stdout.

run this code, then check connection status, 

1
2
????lsof -i -P | grep ree.out
ree.out   12241 xin.wei    3u  IPv4 0xb4e5be7a034326d9      0t0  TCP *:19999 (LISTEN)

client.c

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <netinet/in.h>
#include <arpa/inet.h>

#define BUF_SIZE 1024

void error_handling(char *message);

int main(int argc, char *argv[]) {
    int sock, str_len, recv_len, recv_cnt;
    char message[BUF_SIZE];
    struct sockaddr_in serv_adr;

    if (argc != 3) {
        printf("Usage : %s <IP> <PORT> \n", argv[0]);
        exit(1);
    }

    sock = socket(PF_INET, SOCK_STREAM, 0);
    if (sock == -1) {
        error_handling("socket() error");
    }
    memset(&serv_adr, 0, sizeof(serv_adr));
    serv_adr.sin_family = AF_INET;
    serv_adr.sin_addr.s_addr = inet_addr(argv[1]);
    serv_adr.sin_port = htons(atoi(argv[2]));

    if (connect(sock, (struct sockaddr *) &serv_adr, sizeof(serv_adr)) == -1) {
        error_handling("connect() error !");
    } else {
        puts("Connected.........");
    }

    while (1) {
        fputs("Input message(Q to quit): ", stdout);
        fgets(message, BUF_SIZE, stdin);

        if (!strcmp(message, "q\n") || !strcmp(message, "Q\n")) {
            break;
        }
        str_len = write(sock, message, BUF_SIZE - 1);
        recv_len = 0;
        while (recv_len < str_len) {
            recv_cnt = read(sock, message, BUF_SIZE - 1);
            if (recv_cnt == -1) {
                error_handling("read() error!");
            }
            recv_len += recv_cnt;

        }
        message[str_len] = 0;
        printf("Mesage from server: %s \n", message);

    }
    close(sock);
    return 0;
}


void error_handling(char *message) {
    fputs(message, stderr);
    fputc('\n', stderr);
    exit(1);
}

this client would send the input from stdin to server and echo the response from server.
run the client, then I got:

1
2
3
????lsof -i -P | grep ree.out
ree.out   12241 xin.wei    3u  IPv4 0xb4e5be7a034326d9      0t0  TCP *:19999 (LISTEN)
ree.out   12241 xin.wei    4u  IPv4 0xb4e5be7a03a78181      0t0  TCP localhost:19999->localhost:62228 (ESTABLISHED)

terminate connection by client

let’s terminate the server by press q at client side

1
2
3
4
5
6
7
8
9
10
=> ./ehclient 127.0.0.1 19999
Connected.........
Input message(Q to quit): hello
Mesage from server:
Input message(Q to quit): asdka
Mesage from server: @
Input message(Q to quit): alksdflaskd;afs
Mesage from server:
Input message(Q to quit): q

let’s check status again

1
2
????lsof -i -P | grep ree.out
~/github-personal/algorithm-practice master ????                                                                   ????0|1 5s 20:02:48

the connection totally terminated

terminate connection by ctrl+c on server side

before terminate the server, we need firstly gurantee the connection between client and server established as shown
prevously, then I press ctrl+c to terminate the server
here is the connection status change

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
./time_wait_check.sh
ree.out   14724 xin.wei    3u  IPv4 0xb4e5be7a03423c29      0t0  TCP *:19999 (LISTEN)
ree.out   14724 xin.wei    3u  IPv4 0xb4e5be7a03423c29      0t0  TCP *:19999 (LISTEN)
ree.out   14724 xin.wei    3u  IPv4 0xb4e5be7a03423c29      0t0  TCP *:19999 (LISTEN)
ree.out   14724 xin.wei    3u  IPv4 0xb4e5be7a03423c29      0t0  TCP *:19999 (LISTEN)
ree.out   14724 xin.wei    3u  IPv4 0xb4e5be7a03423c29      0t0  TCP *:19999 (LISTEN)
ree.out   14724 xin.wei    3u  IPv4 0xb4e5be7a03423c29      0t0  TCP *:19999 (LISTEN)
ree.out   14724 xin.wei    4u  IPv4 0xb4e5be7a0385b6d9      0t0  TCP localhost:19999->localhost:62373 (ESTABLISHED)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (ESTABLISHED)
ree.out   14724 xin.wei    3u  IPv4 0xb4e5be7a03423c29      0t0  TCP *:19999 (LISTEN)
ree.out   14724 xin.wei    4u  IPv4 0xb4e5be7a0385b6d9      0t0  TCP localhost:19999->localhost:62373 (ESTABLISHED)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (ESTABLISHED)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSE_WAIT)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSED)
ehclient  14788 xin.wei    3u  IPv4 0xb4e5be7a03a76c29      0t0  TCP localhost:62373->localhost:19999 (CLOSED)

for some reason I don’t see time-wait instead , I saw quite a few close-wait, why?
so I tried with netstat command, I saw this:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63966        FIN_WAIT_2
tcp4       0      0  127.0.0.1.63966        127.0.0.1.19999        CLOSE_WAIT
????netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63966        FIN_WAIT_2
tcp4       0      0  127.0.0.1.63966        127.0.0.1.19999        CLOSE_WAIT
????netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63966        FIN_WAIT_2
tcp4       0      0  127.0.0.1.63966        127.0.0.1.19999        CLOSE_WAIT
????netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63966        FIN_WAIT_2
tcp4       0      0  127.0.0.1.63966        127.0.0.1.19999        CLOSE_WAIT
????netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63966        FIN_WAIT_2
tcp4       0      0  127.0.0.1.63966        127.0.0.1.19999        CLOSE_WAIT
????netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63966        FIN_WAIT_2
tcp4       0      0  127.0.0.1.63966        127.0.0.1.19999        CLOSE_WAIT
????netstat -nat | grep 19999
????netstat -nat | grep 19999

I just saw FIN_WAIT_2 for server and CLOSE_WAIT for client. looks like client not sending FIN to server, so server
keeps staying in FIN_WAIT_2

if we observe closely, we will see CLOSE_WAIT is for client - ehclient, the server side is already terminated, so
the connection status is not presented.
time-wait vs close-wait%20has%20closed%20the%20connection.)

CLOSE_WAIT indicates that the remote endpoint (other side of the connection) has closed the connection.
TIME_WAIT indicates that local endpoint (this side) has closed the connection.
during the close-wait period, if you try to restart server, you would see error

1
2
3
4
5
6
ree.out 19999
bind() error
ree.out 19999
bind() error
ree.out 19999
bind() error

ctrl+c client side

start server and client, check the connection

1
2
3
4
lsof -i -P | grep 19999
ree.out   15314 xin.wei    3u  IPv4 0xb4e5be7a03d426d9      0t0  TCP *:19999 (LISTEN)
ree.out   15314 xin.wei    4u  IPv4 0xb4e5be7a03d486d9      0t0  TCP localhost:19999->localhost:62536 (ESTABLISHED)
ehclient  15327 xin.wei    3u  IPv4 0xb4e5be7a0385b6d9      0t0  TCP localhost:62536->localhost:19999 (ESTABLISHED)

so the client socket would use 62536,
press ctrl+c on client side

1
2
3
4
5
6
7
8
9
time_wait_check.sh
ree.out   15314 xin.wei    4u  IPv4 0xb4e5be7a03d486d9      0t0  TCP localhost:19999->localhost:62536 (ESTABLISHED)
ehclient  15327 xin.wei    3u  IPv4 0xb4e5be7a0385b6d9      0t0  TCP localhost:62536->localhost:19999 (ESTABLISHED)
ree.out   15314 xin.wei    4u  IPv4 0xb4e5be7a03d486d9      0t0  TCP localhost:19999->localhost:62536 (ESTABLISHED)
ehclient  15327 xin.wei    3u  IPv4 0xb4e5be7a0385b6d9      0t0  TCP localhost:62536->localhost:19999 (ESTABLISHED)
ree.out   15314 xin.wei    4u  IPv4 0xb4e5be7a03d486d9      0t0  TCP localhost:19999->localhost:62536 (ESTABLISHED)
ehclient  15327 xin.wei    3u  IPv4 0xb4e5be7a0385b6d9      0t0  TCP localhost:62536->localhost:19999 (ESTABLISHED)
ree.out   15314 xin.wei    4u  IPv4 0xb4e5be7a03d486d9      0t0  TCP localhost:19999->localhost:62536 (ESTABLISHED)
ehclient  15327 xin.wei    3u  IPv4 0xb4e5be7a0385b6d9      0t0  TCP localhost:62536->localhost:19999 (ESTABLISHED)

after ctrl+c the connection just gone.

ctrl+c on server side and then press q on client side

after I do this, I finally saw time_wait

1
2
3
4
5
6
7
8
9
10
11
12
13
14
netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63995        TIME_WAIT
netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63995        TIME_WAIT
netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63995        TIME_WAIT
netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63995        TIME_WAIT
netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63995        TIME_WAIT
netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63995        TIME_WAIT
netstat -nat | grep 19999
tcp4       0      0  127.0.0.1.19999        127.0.0.1.63995        TIME_WAIT

4 times handshake

close-wait
reference
at above pic, host 1 is like our server, host 2 is client.

  1. ctrl+c , the server would send a fin to client, its own status changes to FIN_WAIT_1
  2. client recieve message, when it sees fin, it ACK server immedialtly and change status from
    ESTABLISHED to CLOSE_WAIT
  3. after getting ack from client, serve knows client already got its fin message, so it changes FIN_WAIT_1 to FIN_WAIT_2
  4. the client still not able to close connection now, because its reading or writing buffer may still has remaining
    contents it needs to process. when client read EOF in buffer, then it knows server side it ready to close so it send
    FIN to server, and change its status from CLOSE_WAIT to LAST_ACK
  5. when server got the FIN, it would ACK to client, and changes its status from FIN_WAIT_2 to TIME_WAIT. when
    client get it, then client would close the connection.
  6. the server would wait for 2 MSL(maximum segment lifetime), after that, its status changes from TIME_WAIT to closed

why we nees step 6? because at step 5, server would send ACK to client, if server closed connection immediately but client misses this message, then client would never recieves ACK again , it would keep sending FIN to server and expect ACK. so we give a time_wait for buffering, the server side would keep sending ACK to client until time-wait passed 2 MSL.

summary

  • no matter client or server side, the side who initialize the FIN, ie, attempt to close connection would experience the TIME_WAIT status.
  • the counterpart side would experiences CLOSE_WAIT status
  • if client intialize connection close, then client would have TIME_WAIT and server side would have CLOSE_WAIT, ie,if we see
    a lot of CLOSE_WAIT on remote server, we should know the server side doesn’t give FIN to client, so the server side
    code must have something wrong.
  • if server initialize connection close operation, and we see a lot TIME_WAIT, it might because the client doesn’t get
    ACK from server, or server doesn’t send ACK to client, then client resend FIN to server, so server reset the TIME_WAIT timer.