xfs
[Top] [All Lists]

Re: [BUG] Call trace during snapshot start/stop sequence

To: Dave Chinner <david@xxxxxxxxxxxxx>
Subject: Re: [BUG] Call trace during snapshot start/stop sequence
From: Arkadiusz BubaÅa <arkadiusz.bubala@xxxxxxxxxx>
Date: Thu, 28 Nov 2013 11:00:34 +0100
Cc: xfs@xxxxxxxxxxx
Delivered-to: xfs@xxxxxxxxxxx
In-reply-to: <20131127230608.GJ10988@dastard>
References: <5295C307.6030804@xxxxxxxxxx> <20131127221923.GI10988@dastard> <20131127230608.GJ10988@dastard>
User-agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.16) Gecko/20120613 Icedove/3.0.11
Hello,
thank you for valuable information.

On 28.11.2013 00:06, Dave Chinner wrote:

Running a custom built 3.4.63 kernel with a bunch of out of tree
modules installed. can you reproduce this on a vanilla 3.12 kernel?

Ok, we'll try.

The script is full of bugs, and i don't have time to debug it - it
hard codes /dev/sda in places despite taking the device as a CLI
parameter. It has hard coded mount points.  It sometimes fails to
make the filesystem on the base LV after it's been created.
start_snap() appears to fail for some reason, as it doesn't result
in mounted snapshots. stop_snap fails as well:

Starting snap19 : Thursday 28 November  10:01:26 EST 2013
   Logical volume lv1+snap19 converted to snapshot.
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ OK ] lv1+snap19 activated.
Starting time : 37 s.
---------------------------
Stopping snap0 : Thursday 28 November  10:02:06 EST 2013
[ FAIL ] Can't umount snapshot
[ FAIL ] Can't remove snapshot
[ FAIL ] lv0+snap00 still active !!!
[ OK ] lv0+snap00 umounted.
Stopping time : 0 s.

I've got no idea is this is intended behaviour, but it sure doesn't
seem right to me...


Yes, sometimes umount and remove operations fail. This script tests system stability and these messages are debug info only.
I've fixed it. Now it takes two parameters: device and mount point.

#!/bin/bash

DEV=$1
MOUNTPOINT=$2

function print_usage()
{
    echo "Usage: $0 device mountpoint"
    exit 1;
}

if [ -z "$DEV" ]; then
    print_usage
fi

if [ -z "$MOUNTPOINT" ]; then
    print_usage
fi

function overload()
{
COUNT=$1
temp_COUNT=$COUNT;

while [ -f ./run ]; do
    while [ $COUNT -ge 1 ]; do
        if [ -f ./run ]; then
dd bs=1024 count=102400 if=/dev/zero of=/$2/"_"$COUNT &> /dev/null
        fi;
        let COUNT=$COUNT-1
    done;

    rm $2/*;
    COUNT=$temp_COUNT;

done;
}


function create_vg()
{
    #create physical volume
    pvcreate $DEV
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Unable to create physical volume"
        exit 1
    fi
    #create volume group
    vgcreate -v -s 32M $VG $DEV
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Unable to create volume group"
        exit 1
    fi
}

function create_lv()
{
    local LV="$1"
    #create logical volume
    lvcreate -l 500 -n "$VG+$LV" "$VG"
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Unable to create LV"
        exit 1
    fi
    mkfs -t xfs -f -l lazy-count=0 /dev/$VG/"$VG+$LV" &>/dev/null
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Can't create filesystem"
        exit 1
    fi
}

function create_snapshots()
{
    for ((i=0; i < 20; i++)); do
        if [[ $i -lt 10 ]]; then
            lvcreate -l "64" -n "snap0$i" "$VG"
            if [[ $? -gt 0 ]]; then
                echo "[ FAIL ] Unable to create snapshot LV"
                exit 1
            fi
        else
            lvcreate -l "64" -n "snap$i" "$VG"
            if [[ $? -gt 0 ]]; then
                echo "[ FAIL ] Unable to create snapshot LV"
                exit 1
            fi
        fi
    done
}

function assign_snapshots()
{
    for ((i=0; i < 20; i++)); do
        if [[ $i -lt 10 ]]; then
            lvrename "$VG" "snap0$i" "lv0+snap0$i"
            if [[ $? -gt 0 ]]; then
                echo "[ FAIL ] Unable to rename snapshot LV"
                exit 1
            fi
        else
            lvrename "$VG" "snap$i" "lv1+snap$i"
            if [[ $? -gt 0 ]]; then
                echo "[ FAIL ] Unable to rename snapshot LV"
                exit 1
            fi
        fi
    done
}

function mount_volume()
{
    local MVG=$1
    local MLV=$2
    mkdir -p "$MOUNTPOINT/$MVG+$MLV"
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Unable to create mounting point"
        exit 1
    fi
mount -t xfs -o defaults,usrquota,grpquota,nouuid,noatime,nodiratime "/dev/$MVG/$MVG+$MLV" "$MOUNTPOINT/$MVG+$MLV"
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Unable to mount LV"
        exit 1
    fi
}

function start_overload()
{
    touch ./run
    mkdir -p "$MOUNTPOINT/$1+$2/test"
    overload 50 "$MOUNTPOINT/$1+$2/test" $3 &
    echo "overload $1 $MOUNTPOINT/$1+$2/test $3 &"
    sleep 4;
    echo "[ OK ] copying files to $2 started"
}

function get_snapshot_status()
{
    lvdisplay /dev/$1/$2 | awk ' $0~"LV snapshot status" { print $4 } '
}

function remove_snapshot()
{
    local LVG=$1
    local LLV=$2
    local LSNAP=$3
    umount "$MOUNTPOINT/$LSNAP"
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Can't umount snapshot"
    fi
    lvremove -sf  "/dev/$LVG/$LSNAP"
    if [[ $? -gt 0 ]]; then
        echo "[ FAIL ] Can't remove snapshot"
    fi
}

function create_snapshot()
{
    local LVG=$1
    local LLV=$2
    local LSNAP=$3
    for((it=0; it<7; it++)); do
        local ERROR=0
        local STATUS=`get_snapshot_status $LVG $LSNAP`
        if [[ "$STATUS" == "active" ]]; then
            remove_snapshot $LVG "$LLV+$LSNAP"
        fi
        STATUS=`get_snapshot_status $LVG $LSNAP`
        if [[ "$STATUS" == "active" ]]; then
            remove_snapshot  $LVG "$LLV+$LSNAP"
        fi
        CHUNKSIZE=512
        for ((ile=0;ile<it/2;ile++)); do
            CHUNKSIZE=$((CHUNKSIZE/2))
        done
        lvconvert -s -c $CHUNKSIZE "/dev/$LVG/$LVG+$LLV" "/dev/$LVG/$LSNAP"
        if [[ $? -gt 0 ]]; then
            ERROR=1
        fi
        #mount snapshot
        mkdir -p "$MOUNTPOINT/$LSNAP"
mount -t xfs -o nouuid,noatime "/dev/$LVG/$LSNAP" "$MOUNTPOINT/$LSNAP"
        if [[ $? -gt 0 ]]; then
            ERROR=2
        fi
        create_time=`date "+%Y-%m-%d %H:%M:%S"`
        if [ $ERROR -ne 0 ]; then
            remove_snapshot $LVG $LLV $LSNAP
            sleep 5
        else
            break;
        fi
    done

}

function start_snap()
{
    local i;
    for((i=0; i<20; i++)); do
        echo "Starting snap$i : `date`"
        local START=`date +%s`
        if [[ $i -lt 10 ]]; then
        snapname="lv0+snap0"$i
        create_snapshot $VG "lv0" $snapname
        else
            snapname="lv1+snap"$i
        create_snapshot $VG "lv1" $snapname
        fi
        if [ -z "`lvs | grep $snapname | grep $VG+lv`" ]; then
            echo "[ FAIL ] $snapname not activated !!!"
        else
            echo "[ OK ] $snapname activated."
        fi
        if [ -z "`mount | grep $snapname`" ]; then
            echo "[ FAIL ] $snapname not mounted !!!" >> $LOGFILE
        else
            echo "[ OK ] $snapname mounted."
        fi
        local STOP=$[`date +%s`-$START]
        echo "Starting time : $STOP s."
        echo "---------------------------"
        sleep 2
    done;
}

function stop_snap()
{
    local i
    for((i=0; i<20; i++)); do
        echo "Stopping snap$i : `date`"
        local START=`date +%s`
        if [[ $i -lt 10 ]]; then
            snapname="lv0+snap0"$i
            remove_snapshot $VG "lv0" $snapname
        else
            snapname="lv1+snap"$i
            remove_snapshot $VG "lv1" $snapname
        fi
        if [ "`lvs | grep $snapname | grep $VG+lv`" ]; then
            echo "[ FAIL ] $snapname still active !!!"
        else
            echo "[ OK ] $snapname deactivated."
        fi;
        if [  "`mount | grep $snapname`" ]; then
            echo "[ FAIL ] $snapname still mounted !!!" >> $LOGFILE
        else
            echo "[ OK ] $snapname umounted."
        fi;
        local STOP=$[`date +%s`-$START]
        echo "Stopping time : $STOP s."
        echo "---------------------------"
        sleep 2
    done;
}

VG="vg0"

echo "-------- Creating $VG on $DEV..."
create_vg
echo "[ OK ] Volume group created successfully"
echo "-------- Creating logical volumes on $VG..."
create_lv "lv0"
create_lv "lv1"
echo "[ OK ] Logical volumes created successfully"
echo "-------- Mounting logical volumes..."
mount_volume "$VG" "lv0"
mount_volume "$VG" "lv1"
echo "[ OK ] Logical volumes mounted successfully"
echo "-------- Creating snapshots..."
create_snapshots
echo "[ OK ] Snapshots created successfully"
echo "-------- Assigning snapshots..."
assign_snapshots
echo "[ OK ] Snapshots assigned successfully"
echo "-------- Start overload..."
start_overload "$VG" "lv0"
start_overload "$VG" "lv1"

while true; do
    start_snap 2> /dev/null
    stop_snap  2> /dev/null
done

rm ./run

--
Best regards
Arkadiusz BubaÅa
Open-E Poland Sp. z o.o.
www.open-e.com

<Prev in Thread] Current Thread [Next in Thread>