Prototype d'analyseur de log nfcapd, pour un reporting quotidien, et détermination de saturations sur les liens IP.

chef_d_orchestre.sh 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446
  1. #!/bin/bash
  2. #-------------------------------------------------------------------------------
  3. RS=/usr/bin/rsync # better copy
  4. NBCPU=4 # nombre de threads à lancer. correspond au nombre de CPU/cœurs. pair de préférence.
  5. res=0 # cmd ok/nok ?
  6. DEBUG=0 # 0 : affiche des messages de débug, 1 : stdout = logfile, 2 quiet
  7. DIR1=`grep 1000 /etc/passwd | cut -d':' -f1` # répertoire utilisateur
  8. DIR2="prod" # répertoire pour les programmes
  9. DIR3="sandbox" # répertoire de travail
  10. AJD=`date +"%Y-%m-%d"` # ex: 2016-09-18...
  11. LOGIN="yaplog" # login for mail for SMTP server
  12. NDD="example.org" # domain name
  13. MDPMAIL="MyVerySecureChain" # passwd
  14. SMTPSERVER="smtp.example.org" # where ? gmail/yahoo/outlook
  15. SMTPPORT=587 # which port?
  16. DSTMAIL="yaplogreports" # dstmail@ndd
  17. MAILOBJ="[YaPLog]: Reporting done ${AJD}" # Subject of mail
  18. MAILMSG="Bonjour,\n\nYaPLog a fini de tourner.\nLes logs sont disponibles dans /var/log/yaplog/ et en PJ\nCordialement,\n\nYaPLog." # Body of mail
  19. ERR=0 # y a-t-il eu une erreur
  20. #-------------------------------------------------------------------------------
  21. serr () # nok, always defined
  22. {
  23. /bin/echo -n -e "\e[31m" # color red
  24. /bin/echo ${1} # argv[1]
  25. /bin/echo -n -e "\e[0m" # color none
  26. }
  27. if [ "${DEBUG}" = "0" ]
  28. then
  29. # show output
  30. sout () # info
  31. {
  32. /bin/echo -n -e "\e[33m"
  33. /bin/echo ${1}
  34. /bin/echo -n -e "\e[0m"
  35. }
  36. sok () # ok
  37. {
  38. /bin/echo -n -e "\e[32m"
  39. /bin/echo ${1}
  40. /bin/echo -n -e "\e[0m"
  41. }
  42. sunk () # unknown
  43. {
  44. /bin/echo -n -e "\e[34m"
  45. /bin/echo ${1}
  46. /bin/echo -n -e "\e[0m"
  47. }
  48. elif [ "${DEBUG}" = "1" ]
  49. then
  50. # redirect output to logfile
  51. sout ()
  52. {
  53. /bin/echo -n -e "\e[33m" >> ${LOGFILE}
  54. /bin/echo ${1} >> ${LOGFILE}
  55. /bin/echo -n -e "\e[0m" >> ${LOGFILE}
  56. }
  57. sok ()
  58. {
  59. /bin/echo -n -e "\e[32m" >> ${LOGFILE}
  60. /bin/echo ${1} >> ${LOGFILE}
  61. /bin/echo -n -e "\e[0m" >> ${LOGFILE}
  62. }
  63. sunk ()
  64. {
  65. /bin/echo -n -e "\e[34m" >> ${LOGFILE}
  66. /bin/echo ${1} >> ${LOGFILE}
  67. /bin/echo -n -e "\e[0m" >> ${LOGFILE}
  68. }
  69. serr ()
  70. {
  71. /bin/echo -n -e "\e[31m" >> ${LOGFILE}
  72. /bin/echo ${1} >> ${LOGFILE}
  73. /bin/echo -n -e "\e[0m" >> ${LOGFILE}
  74. }
  75. else
  76. # eliminate most of output
  77. sout ()
  78. {
  79. : # None
  80. }
  81. sok ()
  82. {
  83. : # None
  84. }
  85. sunk ()
  86. {
  87. : # None
  88. }
  89. fi
  90. #-------------------------------------------------------------------------------
  91. # teste si fichier de log ajd existe
  92. NRUNAJD=0
  93. ENDLOOP=0
  94. LOGFILE=0
  95. while :
  96. do
  97. if [ "${ENDLOOP}" = "1" ]
  98. then
  99. # has it been executed today?
  100. if [ "${NRUNAJD}" = "0" ]
  101. then
  102. sok "OK: YaPLog n'a pas encore tourné aujourd'hui." # pas exécuté
  103. else
  104. sunk "Warn: YaPLog a déjà été exécuté ${NRUNAJD} fois aujourd'hui." # au moins une fois
  105. fi
  106. break # "while 1"
  107. fi
  108. if [ -f /var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log ]
  109. then
  110. NRUNAJD=$(($NRUNAJD + 1))
  111. else
  112. touch "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log"
  113. LOGFILE="/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log"
  114. ENDLOOP=1
  115. fi
  116. done
  117. #-------------------------------------------------------------------------------
  118. # sandbox vide ?
  119. NBENTDIR=`ls -lA /home/${DIR1}/${DIR2}/${DIR3} | wc -l` # nombre d'entrées dans le répertoire + 1.
  120. if [ "${NBENTDIR}" = "1" ]
  121. then
  122. :
  123. sok "Ok, la sandbox est vide." # ok, rien dans le répertoire de travail.
  124. else
  125. serr "Le répertoire de travail 'sandbox' n'est pas vide" # répertoire de travail non vide. arrêt.
  126. exit 1
  127. fi
  128. #-------------------------------------------------------------------------------
  129. # Copy
  130. res=0
  131. sout "Copy..."
  132. ${RS} -Phavuz /home/sewan/archive/*.zip /home/administrateur/prod/sandbox/
  133. res=$(($res + $?))
  134. if [ "$res" = "0" ]
  135. then
  136. sok "Copy ok."
  137. else
  138. serr "Copy NOK: $res"
  139. serr "1 : les permission sur les archives sont-elles ok ? : \$ ls -lrthp --color=always --time-style=long-iso # pour s'en assurer"
  140. serr "2 : le disque est-il plein ? : \$ df -h --si | grep -e Size -e /home # pour s'en assurer"
  141. exit 1
  142. fi
  143. ERR=$(($ERR + $res))
  144. #-------------------------------------------------------------------------------
  145. # Mov
  146. res=0
  147. sout "Mov..."
  148. for file in /home/administrateur/prod/sandbox/*.zip
  149. do
  150. echo "${file}"
  151. f=`echo ${file} | cut -d'.' -f1 | cut -d'/' -f6`
  152. dir=`echo ${f} | cut -d'-' -f2`
  153. echo "${f}" "${dir}"
  154. mkdir -p /home/administrateur/prod/sandbox/${dir}
  155. mv ${file} /home/administrateur/prod/sandbox/${dir}/${dir}.zip 2> /dev/null
  156. res=$(($res + $?))
  157. done
  158. if [ "$res" = "0" ]
  159. then
  160. sok "Mov ok."
  161. else
  162. serr "Mov NOK: $res"
  163. fi
  164. ERR=$(($ERR + $res))
  165. #-------------------------------------------------------------------------------
  166. # Unzip
  167. res=0
  168. sout "Unzip..."
  169. for i in `cat /home/administrateur/prod/noSite.csv`
  170. do
  171. if [ -d "/home/administrateur/prod/sandbox/site${i}" ]
  172. then
  173. #sok "${i}"
  174. unzip -qq /home/administrateur/prod/sandbox/site${i}/*.zip -d /home/administrateur/prod/sandbox/site${i}/ 2> /dev/null
  175. res=$(($res + $?))
  176. rm -f /home/administrateur/prod/sandbox/site${i}/*.zip
  177. #res=$(($res + $?)) # on ne veut pas les erreurs de rm
  178. else
  179. :
  180. #sunk "${i}"
  181. fi
  182. done
  183. if [ "$res" = "0" ]
  184. then
  185. sok "Unzip ok."
  186. else
  187. serr "Unzip NOK: $res"
  188. fi
  189. ERR=$(($ERR + $res))
  190. #-------------------------------------------------------------------------------
  191. # Bin2ASCII
  192. res=0
  193. sout "Bin2ASCII..."
  194. for i in `cat /home/administrateur/prod/noSite.csv`
  195. do
  196. if [ -d "/home/administrateur/prod/sandbox/site${i}" ]
  197. then
  198. #sok "${i}"
  199. /home/administrateur/prod/scripts/bash/nfdumpall.sh /home/administrateur/prod/sandbox/site${i}/
  200. res=$(($res + $?))
  201. else
  202. :
  203. #sunk "${i}"
  204. fi
  205. done
  206. if [ "$res" = "0" ]
  207. then
  208. sok "Bin2ASCII ok."
  209. else
  210. serr "Bin2ASCII NOK: $res"
  211. fi
  212. ERR=$(($ERR + $res))
  213. #-------------------------------------------------------------------------------
  214. # Concaténation
  215. res=0
  216. sout "Ccat"
  217. for i in `cat /home/administrateur/prod/noSite.csv`
  218. do
  219. if [ -d "/home/administrateur/prod/sandbox/site${i}" ]
  220. then
  221. DATE=`/bin/ls /home/administrateur/prod/sandbox/site${i}/ | grep ascii | head -1 | cut -d'.' -f2 | cut -c1-8`
  222. #sok "${i}"
  223. for heure in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23
  224. do
  225. for min in 00 15 30 45
  226. do
  227. cat /home/administrateur/prod/sandbox/site${i}/ascii.${DATE}-${heure}-${min} >> /home/administrateur/prod/sandbox/site${i}/site${i}.txt
  228. res=$((res + $?))
  229. done
  230. done
  231. else
  232. :
  233. #sunk "${i}"
  234. fi
  235. done
  236. find /home/administrateur/prod/sandbox/ -name "ascii*" -delete
  237. if [ "$res" = "0" ]
  238. then
  239. sok "Ccat ok."
  240. else
  241. serr "Ccat NOK: $res"
  242. fi
  243. ERR=$(($ERR + $res))
  244. #-------------------------------------------------------------------------------
  245. # ASCII2csv
  246. res=0
  247. sout "ASCII2csv"
  248. for i in `cat /home/administrateur/prod/noSite.csv`
  249. do
  250. if [ -d "/home/administrateur/prod/sandbox/site${i}" ]
  251. then
  252. #sok "${i}"
  253. /home/administrateur/prod/preparser/acsvify.out < /home/administrateur/prod/sandbox/site${i}/site${i}.txt > /home/administrateur/prod/sandbox/site${i}/site${i}.csv
  254. res=$((res + $?))
  255. rm -f /home/administrateur/prod/sandbox/site${i}/site${i}.txt
  256. else
  257. :
  258. #sunk "${i}"
  259. fi
  260. done
  261. if [ "$res" = "0" ]
  262. then
  263. sok "ASCII2csv ok."
  264. else
  265. serr "ASCII2csv NOK: $res"
  266. fi
  267. ERR=$(($ERR + $res))
  268. #-------------------------------------------------------------------------------
  269. # csv2res
  270. res=0
  271. sout "csv2res"
  272. for i in `cat /home/administrateur/prod/noSite.csv`
  273. do
  274. if [ "${i}" = "01" ]
  275. then
  276. j=1
  277. elif [ "{i}" = "02" ]
  278. then
  279. j=2
  280. elif [ "{i}" = "03" ]
  281. then
  282. j=3
  283. elif [ "{i}" = "04" ]
  284. then
  285. j=4
  286. elif [ "{i}" = "05" ]
  287. then
  288. j=5
  289. elif [ "{i}" = "06" ]
  290. then
  291. j=6
  292. elif [ "{i}" = "07" ]
  293. then
  294. j=7
  295. elif [ "{i}" = "08" ]
  296. then
  297. j=8
  298. elif [ "{i}" = "09" ]
  299. then
  300. j=9
  301. else
  302. j=${i}
  303. fi
  304. if [ -d "/home/administrateur/prod/sandbox/site${i}" ]
  305. then
  306. sok "${i}"
  307. if [ "${DEBUG}" = "1" ]
  308. then
  309. # redirect to logfile
  310. /home/administrateur/prod/parser/graphEchantTabl.out ${NBCPU} "/home/administrateur/prod/sandbox/site${i}/site${i}.csv" ${j} "/home/administrateur/prod/sandbox/site${i}/res${i}.csv" >> ${LOGFILE}
  311. res=$((res + $?))
  312. else
  313. # show output on stdout
  314. /home/administrateur/prod/parser/graphEchantTabl.out ${NBCPU} "/home/administrateur/prod/sandbox/site${i}/site${i}.csv" ${j} "/home/administrateur/prod/sandbox/site${i}/res${i}.csv"
  315. res=$((res + $?))
  316. fi
  317. rm -f /home/administrateur/prod/sandbox/site${i}/site${i}.csv
  318. else
  319. :
  320. sunk "${i}"
  321. fi
  322. done
  323. if [ "$res" = "0" ]
  324. then
  325. sok "csv2res ok."
  326. else
  327. serr "csv2res NOK: $res"
  328. fi
  329. ERR=$(($ERR + $res))
  330. #-------------------------------------------------------------------------------
  331. # graphs
  332. res=0
  333. for i in `cat /home/administrateur/prod/noSite.csv`
  334. do
  335. if [ -d "/home/administrateur/prod/sandbox/site${i}" ]
  336. then
  337. sok "${i}"
  338. cp -f /home/administrateur/prod/gnuplot/plotgnu.sh /tmp/
  339. echo 'set title "'"site ${i}"'"' >> /tmp/plotgnu.sh
  340. echo 'plot "/home/administrateur/prod/sandbox/site'"${i}/res${i}.csv"'" using 1 with boxes lc rgb "blueviolet"' >> /tmp/plotgnu.sh
  341. /tmp/plotgnu.sh > /home/administrateur/prod/sandbox/site${i}/site${i}.png
  342. res=$((res + $?))
  343. #rm -f /tmp/plotgnu.sh
  344. read attente
  345. else
  346. :
  347. sunk "${i}"
  348. fi
  349. done
  350. if [ "$res" = "0" ]
  351. then
  352. sok "graphs ok."
  353. else
  354. serr "graphs NOK: $res"
  355. fi
  356. ERR=$(($ERR + $res))
  357. #-------------------------------------------------------------------------------
  358. # \n --> \r\n et backup fichier
  359. res=0
  360. /home/administrateur/prod/postparser/n2rn.out < "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log" > "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log.tmp" # tmp file for mail to windows
  361. res=$(($res + $?)) # success ?
  362. cd "/var/log/yaplog"
  363. if [ "${ERR}" = "0" ]
  364. then
  365. # pas erreur. archive.
  366. tar cfvz "./yaplog.${AJD}.${NRUNAJD}.log.ok.tar.gz" "./yaplog.${AJD}.${NRUNAJD}.log"
  367. res=$(($res + $?)) # success ?
  368. else
  369. # il y a eu une ou plusieurs erreurs. archive.
  370. tar cfvz "./yaplog.${AJD}.${NRUNAJD}.log.err.tar.gz" "./yaplog.${AJD}.${NRUNAJD}.log"
  371. res=$(($res + $?)) # success ?
  372. fi
  373. if [ "$?" = "0" ]
  374. then
  375. mv -f "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log.tmp" "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log" # crosoft aime pas les fichiers .tmp
  376. else
  377. serr "Pas réussi à backuper l'archive"
  378. exit 1; # pour ne pas supprimer de fichiers de log.
  379. fi
  380. if [ "$res" = "0" ]
  381. then
  382. sok "n2rn ok."
  383. else
  384. serr "n2rn NOK: $res"
  385. fi
  386. ERR=$(($ERR + $res))
  387. #-------------------------------------------------------------------------------
  388. # compress and send report
  389. res=0
  390. cd "/var/log/yaplog/"
  391. zip "./yaplog.${AJD}.${NRUNAJD}.log.zip" "./yaplog.${AJD}.${NRUNAJD}.log" # compress for mail
  392. res=$(($res + $?)) # success ?
  393. rm -f "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log" # delete log whose backup is in zip sending, and tar.gz stored.
  394. /usr/bin/sendemail -xu ${LOGIN}"@"${NDD} -xp ${MDPMAIL} -s ${SMTPSERVER}":"${SMTPPORT} -f ${LOGIN}"@"${NDD} -t ${DSTMAIL}"@"${NDD} -u ${MAILOBJ} -m ${MAILMSG} -a ${LOGFILE}".zip" > /dev/null # send mail & shut up about it
  395. res=$(($res + $?)) # success ?
  396. rm -f "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log.tmp" "/var/log/yaplog/yaplog.${AJD}.${NRUNAJD}.log.zip" # delete temporary windows files
  397. if [ "$res" = "0" ]
  398. then
  399. sok "Mail ok."
  400. else
  401. serr "Mail NOK: $res"
  402. fi
  403. ERR=$(($ERR + $res))
  404. #-------------------------------------------------------------------------------
  405. # if errors have happened
  406. if [ "$ERR" = "0" ]
  407. then
  408. sok "Tout est ok."
  409. else
  410. serr "YaPLog a eu $res erreur(s)."
  411. fi
  412. #-------------------------------------------------------------------------------
  413. #############
  414. sok "Done." #
  415. #############