LVM Troubleshooting for REL5
Jump to navigation
Jump to search
we had the problem that we lost a 7TB data disk in case of MBR data corruption.
so I tried to analyze the problem by reconstructing the problem.
it shows how lvm works and how lost PVs/LVs/VGs can be restored if only metadata is gone.
Contents
1 create lvm
note:
/dev/hdb is our testing device
we will attach it to given VG
destroy the metadata of /dev/hdb
and then we repair it.
1.1 disk overview
[root@ora01 ~]# fdisk -l Disk /dev/hda: 10.7 GB, 10737418240 bytes 255 heads, 63 sectors/track, 1305 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes Device Boot Start End Blocks Id System /dev/hda1 * 1 13 104391 83 Linux /dev/hda2 14 1305 10377990 8e Linux LVM Disk /dev/hdb: 10.7 GB, 10737418240 bytes 16 heads, 63 sectors/track, 20805 cylinders Units = cylinders of 1008 * 512 = 516096 bytes Disk /dev/hdb doesn't contain a valid partition table
there is no medatata in disk header:
[root@ora01 ~]# dd < /dev/hdb bs=1024 count=6 2>/dev/null| strings
1.2 create physical volume
[root@ora01 ~]# pvcreate /dev/hdb Physical volume "/dev/hdb" successfully created
now, uuid is written into disk header of physical volume
[root@ora01 ~]# dd < /dev/hdb bs=1024 count=6 2>/dev/null| strings LABELONE LVM2 001E5qmTy2AHfuO68MrUljQfwvbSw0INOnG 5` LVM2 x[5A%r0N*>
[root@ora01 ~]# pvs -o pv_name,uuid PV PV UUID /dev/hda2 YVv9JG-brB2-fktw-Fhn2-8v7G-0Rm6-1AXsYp /dev/hdb E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG
1.3 extending the lvm volume group VG0
attach the pv hdb to VG0
[root@ora01 ~]# vgextend VG0 /dev/hdb Volume group "VG0" successfully extended
now, space is added to VG0
[root@ora01 ~]# vgs -o vg_name,uuid,vg_size VG VG UUID VSize VG0 auPJU5-jYf9-uH0f-zFkK-5cYb-qnH0-XbTQGQ 19.84G
now, complete lvm configuration is written to disk header
[root@ora01 ~]# dd < /dev/hdb bs=1024 count=6 2>/dev/null| strings
LABELONE
LVM2 001E5qmTy2AHfuO68MrUljQfwvbSw0INOnG
R f LVM2 x[5A%r0N*>
VG0 {
id = "auPJU5-jYf9-uH0f-zFkK-5cYb-qnH0-XbTQGQ"
seqno = 6
status = ["RESIZEABLE", "READ", "WRITE"]
flags = []
extent_size = 65536
max_lv = 0
max_pv = 0
physical_volumes {
pv0 {
id = "YVv9JG-brB2-fktw-Fhn2-8v7G-0Rm6-1AXsYp"
device = "/dev/hda2"
status = ["ALLOCATABLE"]
flags = []
dev_size = 20755980
pe_start = 384
pe_count = 316
pv1 {
id = "E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG"
device = "/dev/hdb"
status = ["ALLOCATABLE"]
flags = []
dev_size = 20971520
pe_start = 384
pe_count = 319
logical_volumes {
root {
id = "dpABRE-2SmL-42pF-thxK-vBWj-xeW0-nKk7Zz"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 268
type = "striped"
stripe_count = 1 # linear
stripes = [
"pv0", 0
swap {
id = "60xODA-JUTb-8xI3-cVNb-ILKl-Qx1i-RQaIL8"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 48
type = "striped"
stripe_count = 1 # linear
stripes = [
"pv0", 268
# Generated by LVM2 version 2.02.46-RHEL5 (2009-09-15): Thu Mar 18 11:47:40 2010
contents = "Text Format Volume Group"
version = 1
description = ""
creation_host = "ora01" # Linux ora01 2.6.18-164.11.1.el5 #1 SMP Wed Jan 20 07:39:04 EST 2010 i686
creation_time = 1268909260 # Thu Mar 18 11:47:40 2010
1.4 check for consistency
[root@ora01 ~]# vgck VG0
[root@ora01 ~]# pvck /dev/hdb Found label on /dev/hdb, sector 1, type=LVM2 001 Found text metadata area: offset=4096, size=192512
2 destroy lvm
2.1 destroy metadata of physical volume /dev/hdb
[root@ora01 ~]# dd < /dev/zero > /dev/hdb bs=1024 count=5 5+0 records in 5+0 records out 5120 bytes (5.1 kB) copied, 0.0045225 seconds, 1.1 MB/s
see, its it has changed
[root@ora01 ~]# dd < /dev/hdb bs=1024 count=6 2>/dev/null| strings
volumes {
root {
id = "dpABRE-2SmL-42pF-thxK-vBWj-xeW0-nKk7Zz"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 268
type = "striped"
stripe_count = 1 # linear
stripes = [
"pv0", 0
swap {
id = "60xODA-JUTb-8xI3-cVNb-ILKl-Qx1i-RQaIL8"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 48
type = "striped"
stripe_count = 1 # linear
stripes = [
"pv0", 268
# Generated by LVM2 version 2.02.46-RHEL5 (2009-09-15): Thu Mar 18 11:47:40 2010
contents = "Text Format Volume Group"
version = 1
description = ""
creation_host = "ora01" # Linux ora01 2.6.18-164.11.1.el5 #1 SMP Wed Jan 20 07:39:04 EST 2010 i686
creation_time = 1268909260 # Thu Mar 18 11:47:40 2010
3 repair the lvm
3.1 check the lvm configuration backup of VG0
[root@ora01 ~]# cat /etc/lvm/backup/VG0
# Generated by LVM2 version 2.02.46-RHEL5 (2009-09-15): Thu Mar 18 11:47:40 2010
contents = "Text Format Volume Group"
version = 1
description = "Created *after* executing 'vgextend VG0 /dev/hdb'"
creation_host = "ora01" # Linux ora01 2.6.18-164.11.1.el5 #1 SMP Wed Jan 20 07:39:04 EST 2010 i686
creation_time = 1268909260 # Thu Mar 18 11:47:40 2010
VG0 {
id = "auPJU5-jYf9-uH0f-zFkK-5cYb-qnH0-XbTQGQ"
seqno = 6
status = ["RESIZEABLE", "READ", "WRITE"]
flags = []
extent_size = 65536 # 32 Megabytes
max_lv = 0
max_pv = 0
physical_volumes {
pv0 {
id = "YVv9JG-brB2-fktw-Fhn2-8v7G-0Rm6-1AXsYp"
device = "/dev/hda2" # Hint only
status = ["ALLOCATABLE"]
flags = []
dev_size = 20755980 # 9.89722 Gigabytes
pe_start = 384
pe_count = 316 # 9.875 Gigabytes
}
pv1 {
id = "E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG"
device = "/dev/hdb" # Hint only
status = ["ALLOCATABLE"]
flags = []
dev_size = 20971520 # 10 Gigabytes
pe_start = 384
pe_count = 319 # 9.96875 Gigabytes
}
}
logical_volumes {
root {
id = "dpABRE-2SmL-42pF-thxK-vBWj-xeW0-nKk7Zz"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 268 # 8.375 Gigabytes
type = "striped"
stripe_count = 1 # linear
stripes = [
"pv0", 0
]
}
}
swap {
id = "60xODA-JUTb-8xI3-cVNb-ILKl-Qx1i-RQaIL8"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 48 # 1.5 Gigabytes
type = "striped"
stripe_count = 1 # linear
stripes = [
"pv0", 268
]
}
}
}
}
3.2 check if its really broken
see, its corrupted now
[rot@ora01 ~]# vgck VG0 Couldn't find device with uuid 'E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG'.
[root@ora01 ~]# pvck /dev/hdb Could not find LVM label on /dev/hdb
3.3 rewrite lvm metadata of physical volume /dev/hdb
[root@ora01 ~]# pvcreate --restorefile /etc/lvm/backup/VG0 -u E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG /dev/hdb Couldn't find device with uuid 'E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG'. Physical volume "/dev/hdb" successfully created
3.4 restore configuration to volume group
[root@ora01 ~]# vgcfgrestore -f /etc/lvm/backup/VG0 VG0 Restored volume group VG0
3.5 check if its fixed now
[root@ora01 ~]# vgck VG0
[root@ora01 ~]# pvck /dev/hdb Found label on /dev/hdb, sector 1, type=LVM2 001 Found text metadata area: offset=4096, size=192512
