LVM Troubleshooting for REL5

Aus Bitbull Wiki
Wechseln zu: Navigation, Suche


we had the problem that we lost a 7TB data disk in case of MBR data corruption.
so I tried to analyze the problem by reconstructing the problem.
it shows how lvm works and how lost PVs/LVs/VGs can be restored if only metadata is gone.

create lvm

note:
/dev/hdb is our testing device we will attach it to given VG
destroy the metadata of /dev/hdb
and then we repair it.

disk overview

[root@ora01 ~]# fdisk -l

Disk /dev/hda: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes

   Device Boot      Start         End      Blocks   Id  System
/dev/hda1   *           1          13      104391   83  Linux
/dev/hda2              14        1305    10377990   8e  Linux LVM

Disk /dev/hdb: 10.7 GB, 10737418240 bytes
16 heads, 63 sectors/track, 20805 cylinders
Units = cylinders of 1008 * 512 = 516096 bytes

Disk /dev/hdb doesn't contain a valid partition table

there is no medatata in disk header:

[root@ora01 ~]# dd < /dev/hdb  bs=1024 count=6  2>/dev/null| strings 

create physical volume

[root@ora01 ~]# pvcreate /dev/hdb
  Physical volume "/dev/hdb" successfully created

now, uuid is written into disk header of physical volume

[root@ora01 ~]# dd < /dev/hdb  bs=1024 count=6  2>/dev/null| strings 
LABELONE
LVM2 001E5qmTy2AHfuO68MrUljQfwvbSw0INOnG
5` LVM2 x[5A%r0N*>
[root@ora01 ~]# pvs -o pv_name,uuid 
  PV         PV UUID                               
  /dev/hda2  YVv9JG-brB2-fktw-Fhn2-8v7G-0Rm6-1AXsYp
  /dev/hdb   E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG

extending the lvm volume group VG0

attach the pv hdb to VG0

[root@ora01 ~]# vgextend VG0 /dev/hdb
  Volume group "VG0" successfully extended

now, space is added to VG0

[root@ora01 ~]# vgs -o vg_name,uuid,vg_size
  VG   VG UUID                                VSize 
  VG0  auPJU5-jYf9-uH0f-zFkK-5cYb-qnH0-XbTQGQ 19.84G

now, complete lvm configuration is written to disk header

[root@ora01 ~]# dd < /dev/hdb  bs=1024 count=6  2>/dev/null| strings 
LABELONE
LVM2 001E5qmTy2AHfuO68MrUljQfwvbSw0INOnG
R	f LVM2 x[5A%r0N*>
VG0 {
id = "auPJU5-jYf9-uH0f-zFkK-5cYb-qnH0-XbTQGQ"
seqno = 6
status = ["RESIZEABLE", "READ", "WRITE"]
flags = []
extent_size = 65536
max_lv = 0
max_pv = 0
physical_volumes {
pv0 {
id = "YVv9JG-brB2-fktw-Fhn2-8v7G-0Rm6-1AXsYp"
device = "/dev/hda2"
status = ["ALLOCATABLE"]
flags = []
dev_size = 20755980
pe_start = 384
pe_count = 316
pv1 {
id = "E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG"
device = "/dev/hdb"
status = ["ALLOCATABLE"]
flags = []
dev_size = 20971520
pe_start = 384
pe_count = 319
logical_volumes {
root {
id = "dpABRE-2SmL-42pF-thxK-vBWj-xeW0-nKk7Zz"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 268
type = "striped"
stripe_count = 1	# linear
stripes = [
"pv0", 0
swap {
id = "60xODA-JUTb-8xI3-cVNb-ILKl-Qx1i-RQaIL8"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 48
type = "striped"
stripe_count = 1	# linear
stripes = [
"pv0", 268
# Generated by LVM2 version 2.02.46-RHEL5 (2009-09-15): Thu Mar 18 11:47:40 2010
contents = "Text Format Volume Group"
version = 1
description = ""
creation_host = "ora01"	# Linux ora01 2.6.18-164.11.1.el5 #1 SMP Wed Jan 20 07:39:04 EST 2010 i686
creation_time = 1268909260	# Thu Mar 18 11:47:40 2010

check for consistency

[root@ora01 ~]# vgck VG0
[root@ora01 ~]# pvck /dev/hdb
  Found label on /dev/hdb, sector 1, type=LVM2 001
  Found text metadata area: offset=4096, size=192512

destroy lvm

destroy metadata of physical volume /dev/hdb

[root@ora01 ~]# dd < /dev/zero > /dev/hdb bs=1024 count=5
5+0 records in
5+0 records out
5120 bytes (5.1 kB) copied, 0.0045225 seconds, 1.1 MB/s

see, its it has changed

[root@ora01 ~]# dd < /dev/hdb  bs=1024 count=6  2>/dev/null| strings 
volumes {
root {
id = "dpABRE-2SmL-42pF-thxK-vBWj-xeW0-nKk7Zz"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 268
type = "striped"
stripe_count = 1	# linear
stripes = [
"pv0", 0
swap {
id = "60xODA-JUTb-8xI3-cVNb-ILKl-Qx1i-RQaIL8"
status = ["READ", "WRITE", "VISIBLE"]
flags = []
segment_count = 1
segment1 {
start_extent = 0
extent_count = 48
type = "striped"
stripe_count = 1	# linear
stripes = [
"pv0", 268
# Generated by LVM2 version 2.02.46-RHEL5 (2009-09-15): Thu Mar 18 11:47:40 2010
contents = "Text Format Volume Group"
version = 1
description = ""
creation_host = "ora01"	# Linux ora01 2.6.18-164.11.1.el5 #1 SMP Wed Jan 20 07:39:04 EST 2010 i686
creation_time = 1268909260	# Thu Mar 18 11:47:40 2010

repair the lvm

check the lvm configuration backup of VG0

[root@ora01 ~]# cat /etc/lvm/backup/VG0 
# Generated by LVM2 version 2.02.46-RHEL5 (2009-09-15): Thu Mar 18 11:47:40 2010

contents = "Text Format Volume Group"
version = 1

description = "Created *after* executing 'vgextend VG0 /dev/hdb'"

creation_host = "ora01"	# Linux ora01 2.6.18-164.11.1.el5 #1 SMP Wed Jan 20 07:39:04 EST 2010 i686
creation_time = 1268909260	# Thu Mar 18 11:47:40 2010

VG0 {
	id = "auPJU5-jYf9-uH0f-zFkK-5cYb-qnH0-XbTQGQ"
	seqno = 6
	status = ["RESIZEABLE", "READ", "WRITE"]
	flags = []
	extent_size = 65536		# 32 Megabytes
	max_lv = 0
	max_pv = 0

	physical_volumes {

		pv0 {
			id = "YVv9JG-brB2-fktw-Fhn2-8v7G-0Rm6-1AXsYp"
			device = "/dev/hda2"	# Hint only

			status = ["ALLOCATABLE"]
			flags = []
			dev_size = 20755980	# 9.89722 Gigabytes
			pe_start = 384
			pe_count = 316	# 9.875 Gigabytes
		}

		pv1 {
			id = "E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG"
			device = "/dev/hdb"	# Hint only

			status = ["ALLOCATABLE"]
			flags = []
			dev_size = 20971520	# 10 Gigabytes
			pe_start = 384
			pe_count = 319	# 9.96875 Gigabytes
		}
	}

	logical_volumes {

		root {
			id = "dpABRE-2SmL-42pF-thxK-vBWj-xeW0-nKk7Zz"
			status = ["READ", "WRITE", "VISIBLE"]
			flags = []
			segment_count = 1

			segment1 {
				start_extent = 0
				extent_count = 268	# 8.375 Gigabytes

				type = "striped"
				stripe_count = 1	# linear

				stripes = [
					"pv0", 0
				]
			}
		}

		swap {
			id = "60xODA-JUTb-8xI3-cVNb-ILKl-Qx1i-RQaIL8"
			status = ["READ", "WRITE", "VISIBLE"]
			flags = []
			segment_count = 1

			segment1 {
				start_extent = 0
				extent_count = 48	# 1.5 Gigabytes

				type = "striped"
				stripe_count = 1	# linear

				stripes = [
					"pv0", 268
				]
			}
		}
	}
}

check if its really broken

see, its corrupted now

[rot@ora01 ~]# vgck VG0
  Couldn't find device with uuid 'E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG'.
[root@ora01 ~]# pvck /dev/hdb
  Could not find LVM label on /dev/hdb

rewrite lvm metadata of physical volume /dev/hdb

[root@ora01 ~]# pvcreate --restorefile /etc/lvm/backup/VG0 -u E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG /dev/hdb
  Couldn't find device with uuid 'E5qmTy-2AHf-uO68-MrUl-jQfw-vbSw-0INOnG'.
  Physical volume "/dev/hdb" successfully created

restore configuration to volume group

[root@ora01 ~]# vgcfgrestore -f /etc/lvm/backup/VG0 VG0

 Restored volume group VG0

check if its fixed now

[root@ora01 ~]# vgck VG0


[root@ora01 ~]# pvck /dev/hdb
  Found label on /dev/hdb, sector 1, type=LVM2 001
  Found text metadata area: offset=4096, size=192512

links

http://kbase.redhat.com/faq/docs/DOC-3335