source: tests/01120_asteriskstress @ 26ffad7

Last change on this file since 26ffad7 was c5c522c, checked in by Edwin Eefting <edwin@datux.nl>, 8 years ago

initial commit, transferred from cleaned syn3 svn tree
  • Property mode set to 100755
File size: 11.0 KB
RevLine 
[c5c522c]1#!/bin/bash
2
3# (C)DatuX - Released under GPL
4# Asterisk stresstest script
5# Tries to discover asterisk unstability and regressions with sipp and special dialplans.
6# Usefull to find out you're using the wrong gcc/glibc/compiler options.
7# Configuration on target system is replaced with ./asteriskstress/asterisk/*
8
9
10source testlib || exit 1
11
12for PAR in $*; do eval "$PAR=1" &>/dev/null; done
13
14if [ "$ASTERISK_SKIP_STRESS" ]; then
15        echo "Skipping asterisk stress test..."
16        exit 0
17fi
18
19
20DESCRIPTION1="Skip test if we dont have the right option in SYN-3"
21CMD="
22        if [ -e /etc/syn3-version ]; then
23                grep '\"V\"' /etc/webint/REG || exit 1
24        else
25                exit 0
26        fi
27"
28VERIFYEXIT=0
29if ! sshtest; then
30        echo "No voiceserver, test skipped"
31        exit 0
32fi
33
34
35DESCRIPTION1="Check if box has enough memory (512mb)."
36CMD='[ `grep MemTotal /proc/meminfo|egrep -o "([0-9]+)"` -ge 512000 ]'
37VERIFYEXIT=0
38sshtest || exit 1
39
40#Execute a asterisk cli command
41asterisk_cmd()
42{
43        CMD="asterisk -r -x '$ASTERISK_CMD'"
44        DESCRIPTION1="Asterisk CLI command: $ASTERISK_CMD"
45        VERIFYEXIT=0
46        VERIFYPOSITIVE="$ASTERISK_VERIFYPOSTIVE"
47        VERIFYNEGATIVE="$ASTERISK_VERIFYNEGATIVE"
48        sshtest         
49        return $?
50}
51
52#Check if asterisk is crashed or has logged errors
53crashcheck()
54{
55        DESCRIPTION1="Check for asterisk crashes/errors... ($CASE)"
56        CMD="
57                echo 'Asterisk core is still responding via CLI?'
58                echo 'If not responding, either the load is too high or asterisk cant create new threads because of a glibc/gcc incompatiblity. You also need 512 mb memory!'
59                asterisk -r -x 'core show uptime' | grep 'System uptime' || exit 1
60               
61                echo 'Errors in asterisk logs?'
62                cat /var/log/asterisk/full |
63                        grep -v 'Voicemail will be lost.' |
64                        grep -v 'RTCP .. transmission error' |
65                        grep ERROR && exit 1
66        "
67        VERIFYEXIT=0
68        sshtest || exit 1
69}
70
71#Wait untill asterisk crashes or sipp exists, or a timeout has occurred,
72#or on a couple of other optional checks..
73crashwait()
74{
75        (( END=$SECONDS+$MAXTIME ))
76
77        while [ $SECONDS -lt $END ]; do
78                crashcheck || exit 1
79
80                if [ "$ASTERISK_VERIFYPOSTIVE" ]; then
81                        DESCRIPTION1="Asterisk status check ($CASE)"
82                        asterisk_cmd || exit 1
83                        #we only check positive things ONE time, because at the end of the test it might be gone
84                        if [ "$ASTERISK_VERIFYPOSTIVE" ]; then
85                                ASTERISK_VERIFYPOSTIVE=
86                        fi
87                fi
88
89                if [ "$UAC_NOERRORS" == 1 ]; then
90                        DESCRIPTION1="Sipp client has log no errors? ($CASE)"
91                        CMD="   
92                                test -s /root/asteriskstress/*_*_*errors.log || exit 1
93                        "
94                        VERIFYEXIT=1
95                        sshtest || exit 1
96                fi
97
98                DESCRIPTION1="Sipp client has ended? ($CASE)"
99                CMD="
100                        pgrep -f '^sipp.*trace_screen' || exit 1
101                "
102                VERIFYEXIT=1
103                sshtest && return 0
104
105                echo "Waiting until sipp client is done..., timeout after $MAXTIME seconds. ($CASE)"
106                sleep 5
107        done
108        echo "Timeout! Test didnt complete in time!"
109        exit 1
110}
111
112#runs SIPP test scenario for a certain time and check if nothing crashes or acts weird.
113sipprun()
114{
115        #Auto-resume part.
116        #Because these tests take LONG you can run the script with the 'resume' option.
117        RESUMEFILE=".asteriskstress-$IP/`echo \"$CASE\" | md5sum|cut -f1 -d' '`"
118        if [ -e $RESUMEFILE ]; then
119                echo "SKIPPING CASE: $CASE"
120                return 0
121        fi
122       
123        COUNT=0
124        #restart sipp client in case it crashes (happens a lot :()
125        while true; do
126                DESCRIPTION1="Start sipp test client... ($CASE)"
127                CMD="
128                        ulimit -c unlimited
129                        rm /root/asteriskstress/core*
130                        rm /root/asteriskstress/*_*_*.log
131                        ulimit -n 100000
132                        cd /root/asteriskstress
133                        sipp -bind_local -i 127.0.0.1 -mi 127.0.0.1 -ci 127.0.0.1 -recv_timeout 60000 -send_timeout 60000 -bg -nostdin -trace_shortmsg -trace_screen -trace_err $PARS
134                        pgrep -f '^sipp.*trace_screen' || exit 1
135                "
136                VERIFYEXIT=0
137                sshtest || exit 1
138       
139                #give sipp a while to start connecting
140                sleep 5
141       
142                #wait and see what happens...
143                crashwait
144       
145                DESCRIPTION1="sipp client has exited cleanly, without crash? ($CASE)"
146                CMD="
147                        pgrep -f '^sipp.*trace_screen' && exit 1
148                        test -s /root/asteriskstress/*_*_screen.log || exit 1
149                "
150                VERIFYEXIT=0
151                sshtest && break # everything went fine...break the loop
152       
153                (( COUNT++ ))
154                echo "### sipp client crashed! restarting test! attempt number $COUNT ###"
155        done
156               
157        DESCRIPTION1="sipp server still running? ($CASE)"
158        CMD="
159                pgrep -f '^sipp.*uas' || exit 1
160        "
161        VERIFYEXIT=0
162        sshtest || exit 1
163
164        #let asterisk restart convienently, to check for hanging threads etc..
165        if [ "$ASTERISK_RESTARTCONVENIENT" == "1" ]; then
166                restart_convenient
167        fi
168
169
170        #Done, mark resume file
171        touch $RESUMEFILE
172}
173
174restart_all()
175{
176        VERIFYEXIT=0
177        DESCRIPTION1="Copy config, clear environment, restart asterisk"
178        CMD="
179                svc -d /service/asterisk &>/dev/null
180                killall -9 sipp
181                killall -9 safe_asterisk asterisk
182                sleep 1;
183                rm /etc/asterisk/*
184                cp /root/asteriskstress/asterisk/* /etc/asterisk || exit 1
185                rm /var/log/asterisk/full;
186                rm /home/system/asterisk/spool/voicemail/default/1002/INBOX/*
187                rm /home/system/asterisk/spool/voicemail/default/1002/INBOX/.*
188                if which svc &>/dev/null; then
189                        #start on syn3:
190                        echo 'Starting SYN-3 asterisk'
191                        svc -do /service/asterisk
192                else
193                        #start manually
194                        echo 'Starting manual asterisk'
195                        ulimit -n 100000
196                        asterisk
197                fi
198                sleep 3
199        "
200        sshtest || exit 1
201       
202       
203        DESCRIPTION1="Restart sipp server"
204        CMD="
205                ulimit -n 100000
206                sipp -sn uas -cp 18888 -bind_local -i 127.0.0.2 -mi 127.0.0.2 -ci 127.0.0.2 -mp 16000 -p 15060 -bg -nostdin
207                pgrep -f '^sipp.*uas' || exit 1
208        "
209        VERIFYEXIT=0
210        sshtest || exit 1
211
212        #turn of verbose and debugging, to increase performcance and decrease log size
213        ASTERISK_CMD="core set verbose 0"
214        ASTERISK_VERIFYPOSTIVE=
215        asterisk_cmd || exit 1
216       
217        ASTERISK_CMD="core set debug 0"
218        ASTERISK_VERIFYPOSITIVE=
219        asterisk_cmd || exit 1
220
221}
222
223#Let asterisk restart when convienent.
224#This way we can check for hanging threads.
225restart_convenient()
226{
227        CMD="ls -il /var/run/asterisk/asterisk.* > /tmp/oldstatus"
228        sshtest || exit 1
229
230        ASTERISK_CMD="restart when convenient"
231        asterisk_cmd  || exit 1
232
233        (( END=$SECONDS+ $MAXTIME ))
234
235        while [ $SECONDS -lt $END ]; do
236                DESCRIPTION1="Has asterisk restarted yet? ($CASE)"
237                CMD="
238                        ls -il /var/run/asterisk/asterisk.* > /tmp/newstatus
239                        cmp /tmp/oldstatus /tmp/newstatus && exit 1
240                "
241                VERIFYEXIT=0
242                sshtest && return 0
243                echo "Waiting for asterisk to restart when convenient.."
244                sleep 5
245        done
246        echo "Timeout! Test didnt complete in time!"
247        exit 1
248
249}
250
251################################### PREPERATION #################################
252
253if ! [ "$resume" ]; then
254        rm -r .asteriskstress-$IP &>/dev/null
255fi
256mkdir .asteriskstress-$IP &>/dev/null
257
258scptest "asteriskstress" || exit 1
259
260
261
262MAXTIME=3600
263
264################### TEST CASES ##########################################
265
266#start with a clean asterisk environment
267restart_all
268
269
270
271
272CASE="Meetme test without audio. Asterisk will be unable to create new threads if you have the wrong glibc/gcc/kernelmodule. This also tests the dahdi kernel module."
273PARS="-sn uac -s 1000 localhost -users 1000 -m 4000 -d 5000"
274ASTERISK_CMD="show channels"
275ASTERISK_VERIFYPOSTIVE="MeetMe"
276UAC_NOERRORS=
277ASTERISK_RESTARTCONVENIENT= #takes too long for meetme
278sipprun
279#forcefully restart everything, to recover from the heavy meetme test. (takes ages otherwise)
280restart_all
281
282CASE="Meetme test with audio. Causes 100% softinterupts that freese whole box if you have the wrong glibc/gcc/kenelmodule. (This also tests the dahdi kernel module.)"
283PARS="-sn uac_pcap -s 1000 localhost -r 20 -m 2000 "
284ASTERISK_CMD="show channels"
285ASTERISK_VERIFYPOSTIVE="MeetMe"
286UAC_NOERRORS=
287ASTERISK_RESTARTCONVENIENT= #takes too long for meetme
288sipprun
289# forcefully restart everything, to recover from the heavy meetme test. (takes ages otherwise)
290restart_all
291
292
293CASE="Parking lot test"
294PARS="-sn uac_pcap -s 1001 localhost -r 20 -m 2000 "
295ASTERISK_CMD="show channels"
296ASTERISK_VERIFYPOSTIVE="Park"
297UAC_NOERRORS=
298ASTERISK_RESTARTCONVENIENT=1
299sipprun
300
301
302CASE="Voicemail test"
303PARS="-sn uac_pcap -s 1002 localhost -r 10 -m 1000 "
304ASTERISK_CMD="show channels"
305ASTERISK_VERIFYPOSTIVE="VoiceMail"
306UAC_NOERRORS=
307ASTERISK_RESTARTCONVENIENT=1
308sipprun
309
310
311
312#call/second rate and total calls for codec tests
313#keep this low, because sipp client is checked for errors in this test.
314RATE=2
315CALLS=200
316
317CASE="Codec g729 transcoding"
318PARS="-sn uac_pcap -s 209 localhost -r $RATE -m $CALLS "
319ASTERISK_CMD="sip show channels"
320ASTERISK_VERIFYPOSTIVE="g729"
321UAC_NOERRORS=1
322sipprun
323
324CASE="Codec g723 transcoding"
325PARS="-sn uac_pcap -s 201 localhost -r 1 -m 200 "
326ASTERISK_CMD="sip show channels"
327ASTERISK_VERIFYPOSTIVE="g723"
328UAC_NOERRORS=1
329sipprun
330
331# CASE="Codec g722 transcoding"
332# PARS="-sn uac_pcap -s 213 localhost -r 1 -m 200 "
333# ASTERISK_CMD="sip show channels"
334# ASTERISK_VERIFYPOSTIVE="g722"
335# UAC_NOERRORS=1
336# sipprun
337
338
339CASE="Codec g726 transcoding"
340PARS="-sn uac_pcap -s 212 localhost -r $RATE -m $CALLS "
341ASTERISK_CMD="sip show channels"
342ASTERISK_VERIFYPOSTIVE="g726"
343UAC_NOERRORS=1
344ASTERISK_RESTARTCONVENIENT=1
345sipprun
346
347# CASE="Codec ilbc transcoding"
348# PARS="-sn uac_pcap -s 211 localhost -r $RATE -m $CALLS "
349# ASTERISK_CMD="sip show channels"
350# ASTERISK_VERIFYPOSTIVE="ilbc"
351# UAC_NOERRORS=1
352# sipprun
353
354# CASE="Codec speex transcoding"
355# PARS="-sn uac_pcap -s 210 localhost -r $RATE -m $CALLS "
356# ASTERISK_CMD="sip show channels"
357# ASTERISK_VERIFYPOSTIVE="speex"
358# UAC_NOERRORS=1
359# sipprun
360
361
362CASE="Codec lpc10 transcoding"
363PARS="-sn uac_pcap -s 208 localhost -r $RATE -m $CALLS "
364ASTERISK_CMD="sip show channels"
365ASTERISK_VERIFYPOSTIVE="lpc10"
366ASTERISK_RESTARTCONVENIENT=1
367UAC_NOERRORS=1
368sipprun
369
370CASE="Codec slin transcoding"
371PARS="-sn uac_pcap -s 207 localhost -r $RATE -m $CALLS "
372ASTERISK_CMD="sip show channels"
373ASTERISK_VERIFYPOSTIVE="slin"
374ASTERISK_RESTARTCONVENIENT=1
375UAC_NOERRORS=1
376sipprun
377
378CASE="Codec adpcm transcoding"
379PARS="-sn uac_pcap -s 206 localhost -r $RATE -m $CALLS "
380ASTERISK_CMD="sip show channels"
381ASTERISK_VERIFYPOSTIVE="adpcm"
382ASTERISK_RESTARTCONVENIENT=1
383UAC_NOERRORS=1
384sipprun
385
386CASE="Codec g726aal2 transcoding"
387PARS="-sn uac_pcap -s 205 localhost -r $RATE -m $CALLS "
388ASTERISK_CMD="sip show channels"
389ASTERISK_VERIFYPOSTIVE="g726aal2"
390ASTERISK_RESTARTCONVENIENT=1
391UAC_NOERRORS=1
392sipprun
393
394
395CASE="Codec alaw (sipp pcap uses this too, so now actual transcoding is done)"
396PARS="-sn uac_pcap -s 204 localhost -r $RATE -m $CALLS "
397ASTERISK_CMD="sip show channels"
398ASTERISK_VERIFYPOSTIVE="alaw"
399ASTERISK_RESTARTCONVENIENT=1
400UAC_NOERRORS=1
401sipprun
402
403CASE="Codec ulaw transcoding"
404PARS="-sn uac_pcap -s 203 localhost -r $RATE -m $CALLS "
405ASTERISK_CMD="sip show channels"
406ASTERISK_VERIFYPOSTIVE="ulaw"
407ASTERISK_RESTARTCONVENIENT=1
408UAC_NOERRORS=1
409sipprun
410
411CASE="Codec gsm transcoding"
412PARS="-sn uac_pcap -s 202 localhost -r $RATE -m $CALLS "
413ASTERISK_CMD="sip show channels"
414ASTERISK_VERIFYPOSTIVE="gsm"
415ASTERISK_RESTARTCONVENIENT=1
416UAC_NOERRORS=1
417sipprun
418
419
420#probleem 1: na een test die zowel voicemail als normale nummers belt, blijven er een aantal Dials hangen.
421#dus alleen met 1002 en 203 in de stress.csv
422CASE="Random test of everything (heavy and takes a lot of time)"
423PARS="-sf uac_stress.xml -inf stress.csv localhost -users 100 -m 10000 "
424ASTERISK_CMD=
425ASTERISK_VERIFYPOSTIVE=
426ASTERISK_RESTARTCONVENIENT= #kan niet, ivm probleem 1
427UAC_NOERRORS=
428sipprun
429
430
431echo "########################################################"
432echo "All tests successfull"
Note: See TracBrowser for help on using the repository browser.