$ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do cmd="scp ser-template`[ $h = us1 ] && echo -us1`-v4.cfg sona@$h.youroute.net:~"; echo -e "\n$cmd"; $cmd; done scp ser-template-v4.cfg sona@fr1.youroute.net:~ DSA host key for IP address '91.121.66.202' not in list of known hosts. ser-template-v4.cfg 100% 7294 7.1KB/s 00:00 scp ser-template-v4.cfg sona@fr2.youroute.net:~ DSA host key for IP address '91.121.19.149' not in list of known hosts. ser-template-v4.cfg 100% 7294 7.1KB/s 00:00 scp ser-template-v4.cfg sona@fr3.youroute.net:~ DSA host key for IP address '91.121.101.126' not in list of known hosts. ser-template-v4.cfg 100% 7294 7.1KB/s 00:00 scp ser-template-v4.cfg sona@fr4.youroute.net:~ ser-template-v4.cfg 100% 7294 7.1KB/s 00:00 scp ser-template-v4.cfg sona@fr5.youroute.net:~ ser-template-v4.cfg 100% 7294 7.1KB/s 00:00 scp ser-template-v4.cfg sona@dk1.youroute.net:~ ser-template-v4.cfg 100% 7294 7.1KB/s 00:00 scp ser-template-us1-v4.cfg sona@us1.youroute.net:~ ser-template-us1-v4.cfg 100% 7321 7.2KB/s 00:00 $ ssh sona@fr1.youroute.net sona@fr1$ su fr1# cp ser-template-v4.cfg /usr/home/porta-sip/etc/ser.cfg $ ssh sona@fr2.youroute.net sona@fr2$ su fr2# cp ser-template-v4.cfg /usr/home/porta-sip/etc/ser.cfg $ ssh sona@fr3.youroute.net sona@fr3$ su fr3# cp ser-template-v4.cfg /usr/home/porta-sip/etc/ser.cfg $ ssh sona@fr4.youroute.net sona@fr4$ su fr4# cp ser-template-v4.cfg /usr/home/porta-sip/etc/ser.cfg $ ssh sona@fr5.youroute.net sona@fr5$ su fr5# cp ser-template-v4.cfg /usr/home/porta-sip/etc/ser.cfg $ ssh sona@dk1.youroute.net sona@dk1$ su dk1# cp ser-template-v4.cfg /usr/home/porta-sip/etc/ser.cfg $ ssh sona@us1.youroute.net -bash-2.05b$ su Sip# cp ser-template-us1-v4.cfg /usr/home/porta-sip/etc/ser.cfg $ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do echo -e "\n[$h]"; scp sona@$h.youroute.net:/usr/home/porta-sip/etc/ser.cfg ser-template-$h-v4.download.cfg; done 2> /dev/null [fr1] ser.cfg 100% 7294 7.1KB/s 00:00 [fr2] ser.cfg 100% 7294 7.1KB/s 00:01 [fr3] ser.cfg 100% 7294 7.1KB/s 00:01 [fr4] ser.cfg 100% 7294 7.1KB/s 00:00 [fr5] ser.cfg 100% 7294 7.1KB/s 00:00 [dk1] ser.cfg 100% 7294 7.1KB/s 00:00 [us1] ser.cfg 100% 7321 7.2KB/s 00:00 $ u2d *.download.cfg ser-template-dk1-v4.download.cfg: done. ser-template-fr1-v4.download.cfg: done. ser-template-fr2-v4.download.cfg: done. ser-template-fr3-v4.download.cfg: done. ser-template-fr4-v4.download.cfg: done. ser-template-fr5-v4.download.cfg: done. ser-template-us1-v4.download.cfg: done. $ for f in *.download.cfg; do cmd=diff\ `ls *.download.cfg | head -1`\ $f; echo -e "\n$cmd"; $cmd; done diff ser-template-dk1-v4.download.cfg ser-template-dk1-v4.download.cfg diff ser-template-dk1-v4.download.cfg ser-template-fr1-v4.download.cfg diff ser-template-dk1-v4.download.cfg ser-template-fr2-v4.download.cfg diff ser-template-dk1-v4.download.cfg ser-template-fr3-v4.download.cfg diff ser-template-dk1-v4.download.cfg ser-template-fr4-v4.download.cfg diff ser-template-dk1-v4.download.cfg ser-template-fr5-v4.download.cfg diff ser-template-dk1-v4.download.cfg ser-template-us1-v4.download.cfg 19a20 > alias="%%CANONIC_NAME2%%" $ ssh sona@fr1.youroute.net sona@fr1$ su fr1# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-91.121.66.202], shutdown is in progress ... done. Starting PortaSIP[/var/sipenv-91.121.66.202] done. fr1# exit $ ssh sona@fr2.youroute.net sona@fr2$ su fr2# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-91.121.19.149], shutdown is in progress ............ done. Starting PortaSIP[/var/sipenv-91.121.19.149] done. fr2# exit $ ssh sona@fr3.youroute.net sona@fr3$ su fr3# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-91.121.101.126], shutdown is in progress ............... FAILED! Shutdown has not been completed. Stopped services:. Running services:. fr3# top | grep ser$ 97560 root 1 132 0 37076K 3964K RUN 0:25 67.97% ser fr3# kill -9 97560 fr3# top | grep ser$ fr3# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-91.121.101.126], shutdown is in progress . done. Starting PortaSIP[/var/sipenv-91.121.101.126] done. fr3# exit $ ssh sona@fr4.youroute.net sona@fr4$ su fr4# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-91.121.75.124], shutdown is in progress ............... FAILED! Shutdown has not been completed. Stopped services:. Running services:. fr4# top | grep ser$ 69542 root 1 132 0 37076K 3968K RUN 0:25 67.48% ser fr4# kill -9 69542 fr4# top | grep ser$ fr4# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-91.121.75.124], shutdown is in progress . done. Starting PortaSIP[/var/sipenv-91.121.75.124] done. fr4# exit $ ssh sona@fr5.youroute.net sona@fr5$ su fr5# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-91.121.167.75], shutdown is in progress .. done. Starting PortaSIP[/var/sipenv-91.121.167.75] done. fr5# exit $ ssh sona@dk1.youroute.net sona@dk1$ su dk1# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-82.103.128.3], shutdown is in progress ............... FAILED! Shutdown has not been completed. Stopped services:. Running services:. dk1# top | grep ser$ 43781 root 1 101 0 37076K 4108K RUN 1:41 99.02% ser dk1# kill -9 43781 dk1# top | grep ser$ dk1# /var/sipenv-*/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-82.103.128.3], shutdown is in progress . done. Starting PortaSIP[/var/sipenv-82.103.128.3] done. dk1# exit $ ssh sona@us1.youroute.net -bash-2.05b$ su Sip# /var/sipenv-66.234.138.73/etc/rc.d/sip.sh restart Stop PortaSIP[/var/sipenv-66.234.138.73], shutdown is in progress ... done. Starting PortaSIP[/var/sipenv-66.234.138.73] done. Sip# exit $ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do echo -e "\n[$h]"; scp sona@$h.youroute.net:/var/sipenv-\*`[ $h = us1 ] && echo .73`/etc/ser/ser.cfg ser-running-$h-v4.cfg; done 2> /dev/null [fr1] ser.cfg 100% 7040 6.9KB/s 00:00 [fr2] ser.cfg 100% 7040 6.9KB/s 00:00 [fr3] ser.cfg 100% 7050 6.9KB/s 00:00 [fr4] ser.cfg 100% 7040 6.9KB/s 00:00 [fr5] ser.cfg 100% 7040 6.9KB/s 00:00 [dk1] ser.cfg 100% 7030 6.9KB/s 00:00 [us1] ser.cfg 100% 7067 6.9KB/s 00:00 $ u2d ser-running-*.cfg ser-running-dk1-v4.cfg: done. ser-running-fr1-v4.cfg: done. ser-running-fr2-v4.cfg: done. ser-running-fr3-v4.cfg: done. ser-running-fr4-v4.cfg: done. ser-running-fr5-v4.cfg: done. ser-running-us1-v4.cfg: done. $ grep -A3 \"PING ser-running-*.cfg ser-running-dk1-v4.cfg: if (method == "PING") { ser-running-dk1-v4.cfg- sl_send_reply("200", "OK, Alive here"); ser-running-dk1-v4.cfg- break; ser-running-dk1-v4.cfg- } -- ser-running-fr1-v4.cfg: if (method == "PING") { ser-running-fr1-v4.cfg- sl_send_reply("200", "OK, Alive here"); ser-running-fr1-v4.cfg- break; ser-running-fr1-v4.cfg- } -- ser-running-fr2-v4.cfg: if (method == "PING") { ser-running-fr2-v4.cfg- sl_send_reply("200", "OK, Alive here"); ser-running-fr2-v4.cfg- break; ser-running-fr2-v4.cfg- } -- ser-running-fr3-v4.cfg: if (method == "PING") { ser-running-fr3-v4.cfg- sl_send_reply("200", "OK, Alive here"); ser-running-fr3-v4.cfg- break; ser-running-fr3-v4.cfg- } -- ser-running-fr4-v4.cfg: if (method == "PING") { ser-running-fr4-v4.cfg- sl_send_reply("200", "OK, Alive here"); ser-running-fr4-v4.cfg- break; ser-running-fr4-v4.cfg- } -- ser-running-fr5-v4.cfg: if (method == "PING") { ser-running-fr5-v4.cfg- sl_send_reply("200", "OK, Alive here"); ser-running-fr5-v4.cfg- break; ser-running-fr5-v4.cfg- } -- ser-running-us1-v4.cfg: if (method == "PING") { ser-running-us1-v4.cfg- sl_send_reply("200", "OK, Alive here"); ser-running-us1-v4.cfg- break; ser-running-us1-v4.cfg- } $ grep -A3 \"SUBSCRIBE ser-running-*.cfg ser-running-dk1-v4.cfg: if (method == "SUBSCRIBE") { ser-running-dk1-v4.cfg- sl_send_reply("501", "Subscribe Not Implemented"); ser-running-dk1-v4.cfg- break; ser-running-dk1-v4.cfg- } -- ser-running-fr1-v4.cfg: if (method == "SUBSCRIBE") { ser-running-fr1-v4.cfg- sl_send_reply("501", "Subscribe Not Implemented"); ser-running-fr1-v4.cfg- break; ser-running-fr1-v4.cfg- } -- ser-running-fr2-v4.cfg: if (method == "SUBSCRIBE") { ser-running-fr2-v4.cfg- sl_send_reply("501", "Subscribe Not Implemented"); ser-running-fr2-v4.cfg- break; ser-running-fr2-v4.cfg- } -- ser-running-fr3-v4.cfg: if (method == "SUBSCRIBE") { ser-running-fr3-v4.cfg- sl_send_reply("501", "Subscribe Not Implemented"); ser-running-fr3-v4.cfg- break; ser-running-fr3-v4.cfg- } -- ser-running-fr4-v4.cfg: if (method == "SUBSCRIBE") { ser-running-fr4-v4.cfg- sl_send_reply("501", "Subscribe Not Implemented"); ser-running-fr4-v4.cfg- break; ser-running-fr4-v4.cfg- } -- ser-running-fr5-v4.cfg: if (method == "SUBSCRIBE") { ser-running-fr5-v4.cfg- sl_send_reply("501", "Subscribe Not Implemented"); ser-running-fr5-v4.cfg- break; ser-running-fr5-v4.cfg- } -- ser-running-us1-v4.cfg: if (method == "SUBSCRIBE") { ser-running-us1-v4.cfg- sl_send_reply("501", "Subscribe Not Implemented"); ser-running-us1-v4.cfg- break; ser-running-us1-v4.cfg- } $ # As soon as all servers get updated, the tails of last 2'500'000 lines of all sip.log files started to progressively loose the 483 replies. $ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do echo -e "\n[$h]"; ssh sona@$h.youroute.net "tail -2500000 \`ls /var/sipenv-*/log/sip.log | head -1\` | awk '/^SIP\\/2.0/{bad=0} /^SIP\\/2.0 483/{bad=1} /^CSeq:/ && bad==1 {print \$3}' | sort | uniq -c"; done 2> /dev/null [fr1] [fr2] [fr3] 1 REGISTER 105 SUBSCRIBE [fr4] 3 ACK 2 REGISTER 105 SUBSCRIBE [fr5] 35 ACK 42 INVITE 8 NOTIFY 3 OPTIONS 16 REGISTER 1055 SUBSCRIBE [dk1] 2 ACK 3 INVITE 18 REGISTER 624 SUBSCRIBE [us1] $ date Sun Apr 19 18:00:59 WEST 2009 $ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do echo -e "\n[$h]"; ssh sona@$h.youroute.net "tail -2500000 \`ls /var/sipenv-*/log/sip.log | head -1\` | awk '/^SIP\\/2.0/{bad=0} /^SIP\\/2.0 483/{bad=1} /^CSeq:/ && bad==1 {print \$3}' | sort | uniq -c"; done 2> /dev/null; echo; date [fr1] [fr2] [fr3] 1 REGISTER 105 SUBSCRIBE [fr4] 3 ACK 2 REGISTER 105 SUBSCRIBE [fr5] 35 ACK 42 INVITE 8 NOTIFY 3 OPTIONS 16 REGISTER 1055 SUBSCRIBE [dk1] 2 ACK 1 INVITE 8 REGISTER 333 SUBSCRIBE [us1] Sun Apr 19 18:06:00 WEST 2009 $ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do echo -e "\n[$h]"; ssh sona@$h.youroute.net "tail -2500000 \`ls /var/sipenv-*/log/sip.log | head -1\` | awk '/^SIP\\/2.0/{bad=0} /^SIP\\/2.0 483/{bad=1} /^CSeq:/ && bad==1 {print \$3}' | sort | uniq -c"; done 2> /dev/null; echo; date [fr1] [fr2] [fr3] 1 REGISTER 105 SUBSCRIBE [fr4] 1 REGISTER 70 SUBSCRIBE [fr5] 35 ACK 42 INVITE 7 NOTIFY 3 OPTIONS 13 REGISTER 950 SUBSCRIBE [dk1] [us1] Sun Apr 19 18:15:37 WEST 2009 $ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do echo -e "\n[$h]"; ssh sona@$h.youroute.net "tail -2500000 \`ls /var/sipenv-*/log/sip.log | head -1\` | awk '/^SIP\\/2.0/{bad=0} /^SIP\\/2.0 483/{bad=1} /^CSeq:/ && bad==1 {print \$3}' | sort | uniq -c"; done 2> /dev/null; echo; date [fr1] [fr2] [fr3] 70 SUBSCRIBE [fr4] 1 REGISTER 70 SUBSCRIBE [fr5] 35 ACK 42 INVITE 7 NOTIFY 3 OPTIONS 10 REGISTER 840 SUBSCRIBE [dk1] [us1] Sun Apr 19 18:29:29 WEST 2009 $ for h in fr1 fr2 fr3 fr4 fr5 dk1 us1; do echo -e "\n[$h]"; ssh sona@$h.youroute.net "tail -3000000 \`ls /var/sipenv-*/log/sip.log | head -1\` | awk '/^SIP\\/2.0/{bad=0} /^SIP\\/2.0 483/{bad=1} /^CSeq:/ && bad==1 {print \$3}' | sort | uniq -c"; done 2> /dev/null; echo; date [fr1] [fr2] [fr3] [fr4] [fr5] [dk1] [us1] Sun Apr 19 22:12:46 WEST 2009