Home	last update: 2023-09-29 at 06:00:22 CEST

Keilhau.org

Some notes about this site and how its made

crontab

This site is regenerated every hour

# /etc/crontab: system-wide crontab
# Unlike any other crontab you don't have to run the `crontab'
# command to install the new version when you edit this file
# and files in /etc/cron.d. These files also have username fields,
# that none of the other crontabs do.

SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin

# m h dom mon dow user  command
17 *    * * *   root    cd / && run-parts --report /etc/cron.hourly
25 6    * * *   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily )
47 6    * * 7   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly )
52 6    1 * *   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly )
#

# update keilhau.org website every hour
0 * * * * timo /home/timo/databox/DEVelop/keilhau.org/content/./make

Services

Enable the services

sudo cp -av *.service /etc/systemd/system
sudo systemctl enable dstat.service
sudo systemctl enable pingcheck.service
sudo systemctl enable temper.service

sudo systemctl start dstat.service
sudo systemctl start pingcheck.service
sudo systemctl start temper.service

systemctl list-unit-files

Scripts

make script

script called by cron

#!/bin/bash
# make does it all

dir=$(dirname $0)

(cd $dir && ./update_server_info)
(cd $dir && ./update_plot_stats)
(cd $dir && ./update_temp_stats)
(cd $dir && ./update_ping_stats)
(cd $dir && ./update_opendtu_stats)
(cd $dir && ./generate-docs)
(cd $dir && lftp -f ftp-upload)

update_server_info script

System messages

#!/bin/bash
# script to gather system messages and info

dir="server-info"
cat /proc/mdstat > $dir/mdstat.txt
dmesg|egrep -v "USB disconnect"|egrep -v "new low-speed USB device"|tail -n50 > $dir/dmesg.txt
last|head -n50|uniq > $dir/last.txt
uptime > $dir/uptime.txt
#top -n1 > $dir/top.txt
#/sbin/ifconfig wlan0 | grep "inet addr:" | sed -e 's/.*inet addr://' -e 's/  B.*$//' > ip.txt

update_plot_stats script

Update system status

#!/bin/bash
# script to update the dstat/gnuplot stats

samples=$(bc <<< "60*24")       # 1440 samples
dir=$(dirname $0)
allstats="/home/timo/logs/log.dstat"
plotscript="plot_dstat"

stats="/home/timo/logs/log.24h.dstat"
tail -n${samples} $allstats > $stats
cp -a $stats "$dir"

# do plot !
(cd $dir && ./$plotscript)

start_dstat script

Start writing system stats in the background

#!/bin/bash

dir=$(dirname $0)
dstatFile="/home/timo/logs/log.dstat"
dstat -tam --noheaders --noupdate --output $dstatFile 60 > /dev/null

plot_dstat

Create the system stats diagramms using gnuplot

#!/bin/bash
# script for plotting of the dstat logs

plotDir="$(pwd)"
plotFile="$plotDir/current.gnuplot"
dataFile="$plotDir/current.data"
title='Performance'
outDir='diagramms'
logDir='.'

# this is the column which should be plottet
# in the dstat oputput file ( $9 means the 9th column )
column="\$0"
#writeCol="\$12"                # $12 = send
#writeStr="hostA"
writeCol="\$19"                 # $17 = mem used
writeStr="slave"
readCol="\$19"                  # $17 = mem used
#readCol="\$11"                 # $11 = recv
#readStr="HostB"
readStr="master"


# dstat columns
dateCol="\$1"
usrCol="\$3"
sysCol="\$4"
idlCol="\$5"
waiCol="\$6"
hiqCol="\$7"
siqCol="\$8"
readCol="\$9"
writCol="\$10"
recvCol="\$11"
sendCol="\$12"
inCol="\$13"
outCol="\$14"
intCol="\$15"
cswCol="\$16"
usedCol="\$17"
buffCol="\$18"
cachCol="\$19"
freeCol="\$20"

write(){
        what=$1
        echo "$what" >> $plotFile
}


plotImage(){
        file=$1

        #font="FreeMono"
        font="DejaVuSans"
        write "set xtics rotate by -90"
        local fontSize=7
        write "set xtics font '$font, $fontSize'"
        local fontSize=8
        write "set ytics font '$font,$fontSize'"
        write "set terminal png enhanced size 800,200"
        plot $usrCol "cpu_user"
        plot $sysCol "cpu_sys"
        plot $idlCol "cpu_idle"
        plot $waiCol "cpu_wait"
        plot $hiqCol "cpu_hiq"
        plot $siqCol "cpu_siq"
        plot $readCol "disk_read"
        plot $writeCol "disk_write"
        plot $recvCol "net_receive"
        plot $sendCol "net_send"
        plot $inCol "pages_in"
        plot $outCol "pages_out"
        plot $intCol "interrupts"
        plot $cswCol "contend_switches"
        plot $usedCol "mem_used"
        plot $buffCol "mem_buffered"
        plot $cachCol "mem_cached"
        plot $freeCol "mem_free"
}

plot(){
        col=$1
        title=$2
        write "set output '$outDir/${file}_${title}.png'"
        write "plot '$dataFile' using 1:($col) title '$title' ls 1 with lines"
}


plotCpu(){
        write "plot '$dataFile' using 1:($usrCol) title 'cpu user' ls 1 with lines"
        write "plot '$dataFile' using 1:($sysCol) title 'cpu system' ls 1 with lines"
        write "plot '$dataFile' using 1:($idlCol) title 'cpu idle' ls 1 with lines"
        write "plot '$dataFile' using 1:($waiCol) title 'cpu wait' ls 1 with lines"
        write "plot '$dataFile' using 1:($hiqCol) title 'hiq' ls 1 with lines"
        write "plot '$dataFile' using 1:($siqCol) title 'siq' ls 1 with lines"
}

plotDisk(){
        write "plot '$dataFile' using 1:($readCol/1000000) title 'read' ls 1 with lines"
        write "plot '$dataFile' using 1:($writCol/1000000) title 'write' ls 1 with lines"
}

plotNet(){
        write "plot '$dataFile' using 1:($recvCol/1000000) title 'net receive' ls 1 with lines"
        write "plot '$dataFile' using 1:($sendCol/1000000) title 'net send' ls 1 with lines"
}

plotPaging(){
        write "plot '$dataFile' using 1:($inCol) title 'in' ls 1 with lines"
        write "plot '$dataFile' using 1:($outCol) title 'out' ls 1 with lines"
}

plotInts(){
        write "plot '$dataFile' using 1:($intCol) title 'interrupts' ls 1 with lines"
}

plotCtx(){
        write "plot '$dataFile' using 1:($cswCol) title 'content switches' ls 1 with lines"
}

plotMem(){
        write "plot '$dataFile' using 1:($usedCol/1000000) title 'memory used' ls 1 with lines"
        write "plot '$dataFile' using 1:($buffCol/1000000) title 'memory buffered' ls 1 with lines"
        write "plot '$dataFile' using 1:($cachCol/1000000) title 'memory cached' ls 1 with lines"
        write "plot '$dataFile' using 1:($freeCol/1000000) title 'memory free' ls 1 with lines"
}



if [ ! -d "$outDir" ]; then
        mkdir "$outDir"
fi
if [ ! -d "$outDir" ]; then
        printc "cannot create directory $outDir!"
        exit -1
fi

for file in $(ls $logDir/*.dstat); do
        rm $plotFile
        sed -e 's/,/ /'g $file > tmp1   #replace "," with " "
        sed '1,7d'  tmp1 > tmp2         # remove header
        sed '/^$/d' tmp2 > tmp3         # remove empty lines
        sed '/^"/d' tmp3 > $dataFile    # TODO: this is sh*t
        rm tmp*
        fileName=$(basename $file)
        echo "using: $FileName"

        write "set grid"
        write "set style line 1 lt rgb 'black' lw 1"
        write "set xdata time"
        write "set timefmt \"%d-%m %H:%M:%S\""

        # if the file has a special string in it use an different column to plot
        # TODO: code this better ...
        echo $fileName | grep $readStr
        if [[ $? == 0 ]];then
                echo "plotting $readStr graph of $fileName"
                column=$readCol
        fi
        echo $fileName | grep $writeStr
        if [[ $? == 0 ]];then
                echo "plotting $writeStr graph of $fileName"
                column=$writeCol
        fi

        export GDFONTPATH="/usr/share/fonts/dejavu"

        plotImage $fileName
        gnuplot $plotFile
done

update_temp_stats script

Update system environment temperature

#!/bin/bash
# script to update the temper stats

samples=$(bc <<< "60*24")       # 1440 samples
dir=$(dirname $0)
allstats="/home/timo/logs/info.temp"
plotscript="plot_temp"

stats="/home/timo/logs/info.24h.temp"
tail -n${samples} $allstats > $stats
cp -a $stats "$dir"

stats="/home/timo/logs/info.7d.temp"
awk 'NR % 7 == 0' $allstats|tail -n${samples} > $stats
cp -a $stats "$dir"

stats="/home/timo/logs/info.30d.temp"
awk 'NR % 30 == 0' $allstats|tail -n${samples} > $stats
cp -a $stats "$dir"

# do plot !
(cd $dir && ./$plotscript)

start_temper script

Start writing system environment temperature in the background

#!/bin/bash

tempTool="/home/timo/databox/drvNhandbooksNconfigs/temper_usb_thermometer/pcsensor-1.0.0/pcsensor"
dir=$(dirname $0)
log="/home/timo/logs/info.temp"
zzz=60
while [ 1 ]; do $tempTool >> $log; sleep $zzz; done

plot_temp

Create the temperature diagramms using gnuplot

#!/bin/bash
# script to plot the temper usb device logs

plotDir="$(pwd)"
plotFile="$plotDir/current.temper.gnuplot"
dataFile="$plotDir/current.temper.data"
title='Temperature'
outDir='diagramms'
logDir='.'

# dstat columns
dateCol="\$1"
tempStrCol="\$3"
fahrenheitCol="\$4"
celsiusCol="\$5"

write(){
        what=$1
        echo "$what" >> $plotFile
}


plotImage(){
        file=$1
        font="DejaVuSans"
        write "set xtics rotate by -90"
        local fontSize=7
        write "set xtics font '$font, $fontSize'"
        local fontSize=8
        write "set ytics font '$font,$fontSize'"
        write "set terminal png enhanced size 1440,300"
        plot $celsiusCol "degree"
}

plot(){
        col=$1
        title=$2
        write "set output '$outDir/${file}.${title}.png'"
        write "plot '$dataFile' using 1:($col) title '$title' ls 1 with lines"
}


plotCpu(){
        write "plot '$dataFile' using 1:($celsiusCol) title 'Temperature' ls 1 with lines"
}


if [ ! -d "$outDir" ]; then
        mkdir "$outDir"
fi
if [ ! -d "$outDir" ]; then
        printc "cannot create directory $outDir!"
        exit -1
fi

for file in $(ls $logDir/*.temp); do
        sed 's/C//g' $file > $dataFile
        rm $plotFile
        fileName=$(basename $file)
        fileTitle=$(echo $fileName | sed -e 's/_/ /g')
        echo "using: $fileName"
        echo "processing: $fileTitle"

        write "set grid"
        write "set style line 1 lt rgb 'black' lw 1"
        write "set xdata time"
        write "set timefmt \"%Y/%m/%d %H:%M:%S\""

        export GDFONTPATH="/usr/share/fonts/dejavu"

        plotImage $fileName
        gnuplot $plotFile
done

update_ping_stats script

Update system connectivity

#!/bin/bash
# script to update the ping stats

samples=$(bc <<< "60*24")       # 1440 samples
dir=$(dirname $0)
allstats="/home/timo/logs/log.ping"
plotscript="plot_ping"

stats="/home/timo/logs/log.24h.ping"
tail -n${samples} $allstats > $stats
cp -a $stats "$dir"

stats="/home/timo/logs/log.7d.ping"
awk 'NR % 7 == 0' $allstats|tail -n${samples} > $stats
cp -a $stats "$dir"

stats="/home/timo/logs/log.30d.ping"
awk 'NR % 30 == 0' $allstats|tail -n${samples} > $stats
cp -a $stats "$dir"

stats="/home/timo/logs/log.365d.ping"
awk 'NR % 365 == 0' $allstats|tail -n${samples} > $stats
cp -a $stats "$dir"

# do plot !
(cd $dir && ./$plotscript)

start_pingcheck script

Start writing system connectivity in the background

#!/bin/bash

host="8.8.8.8"
pingCmd="ping -c1 $host"
filter="| grep min | sed 's/[^0-9.]*\([0-9.]*\).*/\1/'"
dir=$(dirname $0)
log="/home/timo/logs/log.ping"
zzz=60

while [ 1 ]; do
        ms=$(eval "$pingCmd $filter")
        sleep $zzz
        date=$(date "+%Y/%m/%d %H:%M:%S")
        # if ms == "" the ping to host was lost
        if [[ $ms == "" ]]; then
                echo "$date 0.0" >> $log
                continue
        fi
        echo "$date $ms" >> $log
done

plot_ping

Create the connectivity diagramms using gnuplot

#!/bin/bash
# script to to plot the ping stats

plotDir="$(pwd)"
plotFile="$plotDir/current.ping.gnuplot"
dataFile="$plotDir/current.ping.data"
title='Ping'
outDir='diagramms'
logDir='.'

# dstat columns
dateCol="\$1"
pingCol="\$3"

write(){
        what=$1
        echo "$what" >> $plotFile
}


plotImage(){
        file=$1
        font="DejaVuSans"
    xStart=$(head -c19 $dataFile)
    xEnd=$(tail $dataFile -n1 | head -c19)
        write "set xtics rotate by -90"
        local fontSize=7
        write "set xtics font '$font, $fontSize'"
        local fontSize=8
    write "set format x \"%Y-%m-%d\n%H:%M:%S\""
#    write "set mxtics 24"
    write "set xrange [ \"$xStart\" : \"$xEnd\" ]"
        write "set ytics font '$font,$fontSize'"
        write "set terminal png enhanced size 1440,300"
        plot $pingCol "milliseconds"
}

plot(){
        col=$1
        title=$2
        write "set output '$outDir/${file}.${title}.png'"
        write "plot '$dataFile' using 1:($col) title '$title' ls 1 with lines"
}


if [ ! -d "$outDir" ]; then
        mkdir "$outDir"
fi
if [ ! -d "$outDir" ]; then
        printc "cannot create directory $outDir!"
        exit -1
fi

for file in $(ls $logDir/*.ping); do
        cp $file $dataFile
        rm $plotFile
        fileName=$(basename $file)
        fileTitle=$(echo $fileName | sed -e 's/_/ /g')
        echo "using: $fileName"
        echo "processing: $fileTitle"

        write "set grid"
        write "set style line 1 lt rgb 'black' lw 1"
        write "set xdata time"
        write "set timefmt \"%Y/%m/%d %H:%M:%S\""
        #write "set logscale y"

        export GDFONTPATH="/usr/share/fonts/dejavu"

        plotImage $fileName
        gnuplot $plotFile
done

generate-docs script

Generate all text files using asciidoc

#!/bin/bash
# generates a new version of keilhau.org

dir=$(dirname $0)
#params=" -b html5 -a toc2 -a themedir=~/databox/DEVelop/keilhau.org/content/themes/shiny -a theme=shiny -a data-uri -v"
params=" -b html5 -a theme=flask -v"
generate="asciidoc $params -v"

files=$(ls $dir/*.txt)

for f in ${files[@]}; do
        $generate $f
done

usb temp sensor

Tool to read the temperature

sudo chown root:root pcsensor
sudo chmod u+s pcsensor

Timo Keilhau.org