3:40 a.m.
Hello.
We are running our application in rootless podman.
After some random time (a couple of hours - a couple of weeks), we lose the
network connectivity into the container.
Everything seems to work fine from inside the container to the rest of the
world (yum/dnf, ping, curl), but it looks like the routing stops working
when someone calls from the outside.
I set up a netcat listener (nc -lv), and called it on localhost (worked
fine) and on the tap-interface (long delays if the packet ever returned). I
also set up a tcpdump in a third screen – output below.
bash-4.4$ podman --version
podman version 4.2.0
bash-4.4$ uname -a
Linux podman-container 5.4.17-2136.315.5.el8uek.x86_64 #2 SMP Wed Dec 21
19:38:18 PST 2022 x86_64 x86_64 x86_64 GNU/Linux
bash-4.4$ cat /etc/os-release
NAME="Oracle Linux Server"
VERSION="8.7"
ID="ol"
ID_LIKE="fedora"
VARIANT="Server"
VARIANT_ID="server"
VERSION_ID="8.7"
PLATFORM_ID="platform:el8"
PRETTY_NAME="Oracle Linux Server 8.7"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:oracle:linux:8:7:server"
HOME_URL="https://linux.oracle.com/"
BUG_REPORT_URL="https://bugzilla.oracle.com/"
ORACLE_BUGZILLA_PRODUCT="Oracle Linux 8"
ORACLE_BUGZILLA_PRODUCT_VERSION=8.7
ORACLE_SUPPORT_PRODUCT="Oracle Linux"
ORACLE_SUPPORT_PRODUCT_VERSION=8.7
# Testing communication using 'localhost' inside the container - works as
expected
[root@NC-Test_podman-container /]# nc -lv 10370
Listening on 0.0.0.0 10370
Connection received on localhost 47218
ping from server
ping from client
[root@NC-Test_podman-container /]# nc -v localhost 10370
nc: connect to localhost (::1) port 10370 (tcp) failed: Connection refused
Connection to localhost (127.0.0.1) 10370 port [tcp/*] succeeded!
ping from server
ping from client
# Testing communication using hostname - "some" packets arrives, but only
after a random delay of about 30-600 seconds
[root@NC-Test_podman-container /]# nc -lv 10370
Listening on 0.0.0.0 10370
server
Connection received on podman-container 59258
client
[root@NC-Test_podman-container /]# nc -v podman-container 10370
Connection to podman-container (10.11.12.102) 10370 port [tcp/*] succeeded!
client
server
[root@NC-Test_podman-container base_domain]# tcpdump -vv -X host
podman-container and port 10370
dropped privs to tcpdump
tcpdump: listening on tap0, link-type EN10MB (Ethernet), capture size
262144 bytes
12:41:49.080602 IP (tos 0x0, ttl 64, id 61404, offset 0, flags [DF], proto
TCP (6), length 47)
podman-container.56372 > podman-container-oob.10370: Flags [P.], cksum
0xdb21 (correct), seq 2129174302:2129174309, ack 1071210498, win 65480,
length 7
0x0000: 4500 002f efdc 4000 4006 7df1 0a00 0264 E../..@.@.}....d
0x0010: 0a31 b666 dc34 2882 7ee8 9f1e 3fd9 6002 .1.f.4(.~...?.`.
0x0020: 5018 ffc8 db21 0000 636c 6965 6e74 0a P....!..client.
12:41:49.080783 IP (tos 0x0, ttl 64, id 48821, offset 0, flags [none],
proto TCP (6), length 40)
podman-container-oob.10370 > podman-container.56372: Flags [.], cksum
0x2039 (correct), seq 1, ack 7, win 65535, length 0
0x0000: 4500 0028 beb5 0000 4006 ef1f 0a31 b666 E..(....@....1.f
0x0010: 0a00 0264 2882 dc34 3fd9 6002 7ee8 9f25 ...d(..4?.`.~..%
0x0020: 5010 ffff 2039 0000 P....9..
12:42:28.673431 IP (tos 0x0, ttl 64, id 49091, offset 0, flags [none],
proto TCP (6), length 40)
podman-container-oob.10370 > podman-container.51394: Flags [F.], cksum
0xf92e (correct), seq 946730519, ack 2284994989, win 65535, length 0
0x0000: 4500 0028 bfc3 0000 4006 ee11 0a31 b666 E..(....@....1.f
0x0010: 0a00 0264 2882 c8c2 386d f617 8832 41ad ...d(...8m...2A.
0x0020: 5011 ffff f92e 0000 P.......
12:42:28.673436 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP
(6), length 40)
podman-container.51394 > podman-container-oob.10370: Flags [R], cksum
0x27c1 (correct), seq 2284994989, win 0, length 0
0x0000: 4500 0028 0000 4000 4006 6dd5 0a00 0264 E..(..@.@.m....d
0x0010: 0a31 b666 c8c2 2882 8832 41ad 0000 0000 .1.f..(..2A.....
0x0020: 5004 0000 27c1 0000 P...'...
12:44:28.693154 IP (tos 0x0, ttl 64, id 49943, offset 0, flags [none],
proto TCP (6), length 47)
podman-container-oob.10370 > podman-container.56372: Flags [P.], cksum
0xcadb (correct), seq 1:8, ack 7, win 65535, length 7
0x0000: 4500 002f c317 0000 4006 eab6 0a31 b666 E../....@....1.f
0x0010: 0a00 0264 2882 dc34 3fd9 6002 7ee8 9f25 ...d(..4?.`.~..%
0x0020: 5018 ffff cadb 0000 7365 7276 6572 0a P.......server.
12:44:28.693174 IP (tos 0x0, ttl 64, id 61405, offset 0, flags [DF], proto
TCP (6), length 40)
podman-container.56372 > podman-container-oob.10370: Flags [.], cksum
0x2070 (correct), seq 7, ack 8, win 65473, length 0
0x0000: 4500 0028 efdd 4000 4006 7df7 0a00 0264 E..(..@.@.}....d
0x0010: 0a31 b666 dc34 2882 7ee8 9f25 3fd9 6009 .1.f.4(.~..%?.`.
0x0020: 5010 ffc1 2070 0000 P....p..
Kind regards
//Henrik