星期日, 11月 08, 2015

[原創] 12c RAC 使用LVM mirror ocrdisk的模擬測試

假設我們使用三個voting disk (normal redunancy) ,
votingdisk1 :  /dev/sdh1
votingdisk2 :  /dev/sdi1
votingdisk3 :  /dev/mapper/vg--crsmirror-lv_mirror : /dev/sdj, /dev/sdk

模擬測試使用LVM mirror ocrdisk3 , 刪掉所有OCR ASM disk Header之後修復, 以及修復LVM mirror disk的過程

Create Physical Volume
 pvcreate /dev/sdj /dev/sdk

[root@rac1 ~]# pvcreate /dev/sdj /dev/sdk
  Physical volume "/dev/sdj" successfully created
  Physical volume "/dev/sdk" successfully created

Display the physical volumes:

# pvdisplay

[root@rac1 ~]#  pvdisplay
  --- Physical volume ---
  PV Name               /dev/sda2
  VG Name               vg_rac1
  PV Size               39.51 GiB / not usable 3.00 MiB
  Allocatable           yes (but full)
  PE Size               4.00 MiB
  Total PE              10114
  Free PE               0
  Allocated PE          10114
  PV UUID               wuRltg-6PRJ-ehHd-xxBc-NDMT-sJzw-ksDLU5

  "/dev/sdj" is a new physical volume of "2.00 GiB"
  --- NEW Physical volume ---
  PV Name               /dev/sdj
  VG Name
  PV Size               2.00 GiB
  Allocatable           NO
  PE Size               0
  Total PE              0
  Free PE               0
  Allocated PE          0
  PV UUID               rWOc2k-RRHn-qaf1-cyfp-9ise-yo9r-ukIhs7

  "/dev/sdk" is a new physical volume of "2.00 GiB"
  --- NEW Physical volume ---
  PV Name               /dev/sdk
  VG Name
  PV Size               2.00 GiB
  Allocatable           NO
  PE Size               0
  Total PE              0
  Free PE               0
  Allocated PE          0
  PV UUID               BkdUPV-bDSf-Q0Ps-XSdy-cK47-97pD-7adxaW

[root@rac1 ~]# pvs
  PV         VG      Fmt  Attr PSize  PFree
  /dev/sda2  vg_rac1 lvm2 a--  39.51g    0
  /dev/sdj           lvm2 ---   2.00g 2.00g
  /dev/sdk           lvm2 ---   2.00g 2.00g

Create Volume Group
# vgcreate vg-crsmirror /dev/sdj /dev/sdk

[root@rac1 ~]# vgcreate vg-crsmirror /dev/sdj /dev/sdk
  Volume group "vg-crsmirror" successfully created
[root@rac1 ~]#

[root@rac1 ~]# vgdisplay
  --- Volume group ---
  VG Name               vg-crsmirror
  System ID
  Format                lvm2
  Metadata Areas        2
  Metadata Sequence No  1
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                0
  Open LV               0
  Max PV                0
  Cur PV                2
  Act PV                2
  VG Size               3.99 GiB
  PE Size               4.00 MiB
  Total PE              1022
  Alloc PE / Size       0 / 0
  Free  PE / Size       1022 / 3.99 GiB
  VG UUID               hX8G3p-Ft0n-pYAW-oGWZ-QDnP-RcoA-phC8eN

Create Mirrored Logical Volume
 lvcreate -L 512M -m1 -n lv_mirror vg-crsmirror

[root@rac1 ~]# lvcreate -L 2040M -m1 -n lv_mirror vg-crsmirror    #lvremove /dev/vg-crsmirror/lv_mirror
  Logical volume "lv_mirror" created

Activating LVM
 lvchange -ay /dev/vg-crsmirror/lv_mirror
 #lvchange -an  /dev/vg-crsmirror/lv_mirror

Create LVM On a New Hard Disk
fdisk -l |grep mirror
[root@rac1 vg-crsmirror]# fdisk -l |grep mirror
Disk /dev/mapper/vg--crsmirror-lv_mirror_mlog: 4 MB, 4194304 bytes
Disk /dev/mapper/vg--crsmirror-lv_mirror_mimage_0: 536 MB, 536870912 bytes
Disk /dev/mapper/vg--crsmirror-lv_mirror_mimage_1: 536 MB, 536870912 bytes
Disk /dev/mapper/vg--crsmirror-lv_mirror: 536 MB, 536870912 bytes

[root@rac1 vg-crsmirror]# lvs -a -o +seg_pe_ranges --segments
  LV                   VG           Attr       #Str Type   SSize   PE Ranges  
  lv_mirror            vg-crsmirror mwi-a-m---    2 mirror 512.00m lv_mirror_mimage_0:0-127 lv_mirror_mimage_1:0-127
  [lv_mirror_mimage_0] vg-crsmirror iwi-aom---    1 linear 512.00m /dev/sdj:0-127
  [lv_mirror_mimage_1] vg-crsmirror iwi-aom---    1 linear 512.00m /dev/sdk:0-127
  [lv_mirror_mlog]     vg-crsmirror lwi-aom---    1 linear   4.00m /dev/sdk:128-128
  lv_root              vg_rac1      -wi-ao----    1 linear  39.01g /dev/sda2:0-9985
  lv_swap              vg_rac1      -wi-ao----    1 linear 512.00m /dev/sda2:9986-10113

[root@rac1 vg-crsmirror]# fdisk -l  /dev/mapper/vg--crsmirror-lv_mirror

Disk /dev/mapper/vg--crsmirror-lv_mirror: 536 MB, 536870912 bytes
255 heads, 63 sectors/track, 65 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

-------------------------------------------------------------------------------
在第二台執行的過程

[root@rac2 ~]#  pvcreate /dev/sdj /dev/sdk
  Can't initialize physical volume "/dev/sdj" of volume group "vg-crsmirror" without -ff
  Can't initialize physical volume "/dev/sdk" of volume group "vg-crsmirror" without -ff

[root@rac2 ~]# pvs
  PV         VG           Fmt  Attr PSize  PFree
  /dev/sda2  vg_rac2      lvm2 a--  39.51g    0
  /dev/sdj   vg-crsmirror lvm2 a--   2.00g 1.50g
  /dev/sdk   vg-crsmirror lvm2 a--   2.00g 1.49g
[root@rac2 ~]# vgs
  VG           #PV #LV #SN Attr   VSize  VFree
  vg-crsmirror   2   1   0 wz--n-  3.99g 2.99g
  vg_rac2        1   2   0 wz--n- 39.51g    0
[root@rac2 ~]#

[root@rac2 ~]# lvcreate -L 512M -m1 -n lv_mirror vg-crsmirror
  Logical volume "lv_mirror" already exists in volume group "vg-crsmirror"

[root@rac2 ~]# lvdisplay |grep mirror
  LV Path                /dev/vg-crsmirror/lv_mirror
  LV Name                lv_mirror
  VG Name                vg-crsmirror

[root@rac2 dev]#  lvm pvscan
  PV /dev/sdj    VG vg-crsmirror   lvm2 [2.00 GiB / 1.50 GiB free]
  PV /dev/sdk    VG vg-crsmirror   lvm2 [2.00 GiB / 1.49 GiB free]
  PV /dev/sda2   VG vg_rac2        lvm2 [39.51 GiB / 0    free]
  Total: 3 [43.50 GiB] / in use: 3 [43.50 GiB] / in no VG: 0 [0   ]
[root@rac2 dev]#  lvm vgscan
  Reading all physical volumes.  This may take a while...
  Found volume group "vg-crsmirror" using metadata type lvm2
  Found volume group "vg_rac2" using metadata type lvm2
[root@rac2 dev]# lvm lvscan
  inactive          '/dev/vg-crsmirror/lv_mirror' [512.00 MiB] inherit
  ACTIVE            '/dev/vg_rac2/lv_root' [39.01 GiB] inherit
  ACTIVE            '/dev/vg_rac2/lv_swap' [512.00 MiB] inherit


-------------------------------------------------------------------------------
[root@rac1 ~]# fdisk /dev/mapper/vg--crsmirror-lv_mirror
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel with disk identifier 0x30a357cb.
Changes will remain in memory only, until you decide to write them.
After that, of course, the previous content won't be recoverable.

Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)

WARNING: DOS-compatible mode is deprecated. It's strongly recommended to
         switch off the mode (command 'c') and change display units to
         sectors (command 'u').

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-65, default 1):
Using default value 1
Last cylinder, +cylinders or +size{K,M,G} (1-65, default 65):
Using default value 65

Command (m for help): w
The partition table has been altered!

Calling ioctl() to re-read partition table.

WARNING: Re-reading the partition table failed with error 22: Invalid argument.
The kernel still uses the old table. The new table will be used at
the next reboot or after you run partprobe(8) or kpartx(8)
Syncing disks.
[root@rac1 ~]#


[root@rac1 ~]# fdisk -l |grep mirror
Disk /dev/mapper/vg--crsmirror-lv_mirror_mlog: 4 MB, 4194304 bytes
Disk /dev/mapper/vg--crsmirror-lv_mirror_mimage_0: 536 MB, 536870912 bytes
/dev/mapper/vg--crsmirror-lv_mirror_mimage_0p1               1          65      522081   83  Linux
Disk /dev/mapper/vg--crsmirror-lv_mirror_mimage_1: 536 MB, 536870912 bytes
/dev/mapper/vg--crsmirror-lv_mirror_mimage_1p1               1          65      522081   83  Linux
Disk /dev/mapper/vg--crsmirror-lv_mirror: 536 MB, 536870912 bytes
/dev/mapper/vg--crsmirror-lv_mirrorp1               1          65      522081   83  Linux


[root@rac1 ~]# oracleasm createdisk newocr1 /dev/sdh1
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm createdisk newocr2 /dev/sdi1
Writing disk header: done
Instantiating disk: done

oracleasm createdisk newocr3 /dev/mapper/vg--crsmirror-lv_mirror
Writing disk header: done
Instantiating disk: done
[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm scandisks^C
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
DISK6
NEWOCR1
NEWOCR2
NEWOCR3

-------------------------------------------------------------------------------

[root@rac2 mapper]# lvchange -ay /dev/vg-crsmirror/lv_mirror
[root@rac2 mapper]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
Instantiating disk "NEWOCR3"
[root@rac2 mapper]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
DISK6
NEWOCR1
NEWOCR2
NEWOCR3


---
# 刪掉所有OCR ASM disk Header之後修復
前置作業, 把CRS都停下來之後, 刪除LUN header

 oracleasm deletedisk NEWOCR1
 oracleasm deletedisk NEWOCR2
 oracleasm deletedisk NEWOCR3

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/ocrconfig -showbackup
PROT-26: Oracle Cluster Registry backup locations were retrieved from a local copy

rac1     2015/11/04 18:18:33     /u01/app/12.1.0.2/grid/cdata/racscan/backup00.ocr     3281833642

rac1     2015/11/04 14:18:33     /u01/app/12.1.0.2/grid/cdata/racscan/backup01.ocr     3281833642

rac1     2015/11/04 10:18:33     /u01/app/12.1.0.2/grid/cdata/racscan/backup02.ocr     3281833642

rac1     2015/11/03 14:18:32     /u01/app/12.1.0.2/grid/cdata/racscan/day.ocr     3281833642

rac1     2015/10/31 04:56:42     /u01/app/12.1.0.2/grid/cdata/racscan/week.ocr     3281833642

rac1     2015/03/27 13:42:09     /u01/app/12.1.0.2/grid/cdata/racscan/backup_20150327_134209.ocr     3281833642

[root@rac1 ~]# su - grid
[grid@rac1 ~]$ olsnodes
rac1
rac2

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rac1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rac1'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rac1'
CRS-2673: Attempting to stop 'ora.ctssd' on 'rac1'
CRS-2673: Attempting to stop 'ora.evmd' on 'rac1'
CRS-2673: Attempting to stop 'ora.asm' on 'rac1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rac1'
CRS-2677: Stop of 'ora.drivers.acfs' on 'rac1' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.evmd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.asm' on 'rac1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rac1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rac1'
CRS-2677: Stop of 'ora.cssd' on 'rac1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rac1'
CRS-2677: Stop of 'ora.gipcd' on 'rac1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rac1' has completed
CRS-4133: Oracle High Availability Services has been stopped.

[root@rac2 ~]# /u01/app/12.1.0.2/grid/bin/crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rac2'
CRS-2673: Attempting to stop 'ora.asm' on 'rac2'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rac2'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rac2'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rac2'
CRS-2677: Stop of 'ora.drivers.acfs' on 'rac2' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'rac2' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'rac2' succeeded
CRS-2677: Stop of 'ora.asm' on 'rac2' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rac2'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rac2' succeeded
CRS-2673: Attempting to stop 'ora.ctssd' on 'rac2'
CRS-2673: Attempting to stop 'ora.evmd' on 'rac2'
CRS-2677: Stop of 'ora.ctssd' on 'rac2' succeeded
CRS-2677: Stop of 'ora.evmd' on 'rac2' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rac2'
CRS-2677: Stop of 'ora.cssd' on 'rac2' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rac2'
CRS-2677: Stop of 'ora.gipcd' on 'rac2' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rac2' has completed
CRS-4133: Oracle High Availability Services has been stopped.

/u01/app/12.1.0.2/grid/bin/ocrconfig -restore /u01/app/12.1.0.2/grid/cdata/racscan/backup00.ocr

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/ocrconfig -restore /u01/app/12.1.0.2/grid/cdata/racscan/backup00.ocr
PROT-35: The configured OCR locations are not accessible
PROC-26: Error while accessing the physical storage

Start ASM on one node in exclusive mode.
/u01/app/12.1.0.2/grid/bin/crsctl start crs -excl -nocrs

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/crsctl start crs -excl -nocrs
CRS-4123: Oracle High Availability Services has been started.
CRS-2672: Attempting to start 'ora.evmd' on 'rac1'
CRS-2672: Attempting to start 'ora.mdnsd' on 'rac1'
CRS-2676: Start of 'ora.evmd' on 'rac1' succeeded
CRS-2676: Start of 'ora.mdnsd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'rac1'
CRS-2676: Start of 'ora.gpnpd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'rac1'
CRS-2672: Attempting to start 'ora.gipcd' on 'rac1'
CRS-2676: Start of 'ora.cssdmonitor' on 'rac1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'rac1'
CRS-2672: Attempting to start 'ora.diskmon' on 'rac1'
CRS-2676: Start of 'ora.diskmon' on 'rac1' succeeded
CRS-2676: Start of 'ora.cssd' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2672: Attempting to start 'ora.ctssd' on 'rac1'
CRS-2676: Start of 'ora.ctssd' on 'rac1' succeeded
CRS-2676: Start of 'ora.cluster_interconnect.haip' on 'rac1' succeeded
CRS-2672: Attempting to start 'ora.asm' on 'rac1'
CRS-2676: Start of 'ora.asm' on 'rac1' succeeded

Check CRSD is down and stop it if it is not.

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/crsctl status resource ora.crsd -init
NAME=ora.crsd
TYPE=ora.crs.type
TARGET=OFFLINE
STATE=OFFLINE

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   c86457b0b3234f29bffc5d814e92df63 (ORCL:DISK1) [DATA]
Located 1 voting disk(s).
[root@rac1 ~]#

[grid@rac1 ~]$ sqlplus / as sysasm

SQL*Plus: Release 12.1.0.2.0 Production on Sun Nov 8 02:03:30 2015

Copyright (c) 1982, 2014, Oracle.  All rights reserved.


Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options

SQL> -- drop diskgroup newocr force including contents;
     --  alter diskgroup newocr rebalance; 如果 drop不掉記得先rebalance

SQL> create diskgroup newocr normal redundancy disk 'ORCL:NEWOCR1', 'ORCL:NEWOCR2', 'ORCL:NEWOCR3' attribute 'compatible.asm' = '12.1';
create diskgroup newocr normal redundancy disk 'ORCL:NEWOCR1', 'ORCL:NEWOCR2', 'ORCL:NEWOCR3' attribute 'compatible.asm' = '12.1'
*
ERROR at line 1:
ORA-15018: diskgroup cannot be created
ORA-15410: Disks in disk group NEWOCR do not have equal size.

SQL>
 create diskgroup OCRNEW normal redundancy disk 'ORCL:NEWOCR1' size 2000M, 'ORCL:NEWOCR2'  size 2000M, 'ORCL:NEWOCR3'  size 2000M attribute 'compatible.asm' = '11.2';

SQL>

Diskgroup created.

SQL>

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/ocrconfig -restore /u01/app/12.1.0.2/grid/cdata/racscan/backup00.ocr
[root@rac1 ~]#

[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          4
         Total space (kbytes)     :     409568
         Used space (kbytes)      :       1420
         Available space (kbytes) :     408148
         ID                       : 2056011687
         Device/File Name         :    +OCRNEW
                                    Device/File integrity check succeeded

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

         Cluster registry integrity check succeeded

         Logical corruption check succeeded

[root@rac1 ~]#
[root@rac1 ~]# /u01/app/12.1.0.2/grid/bin/crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rac1'
CRS-2673: Attempting to stop 'ora.storage' on 'rac1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rac1'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rac1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rac1'
CRS-2677: Stop of 'ora.storage' on 'rac1' succeeded
CRS-2673: Attempting to stop 'ora.ctssd' on 'rac1'
CRS-2673: Attempting to stop 'ora.evmd' on 'rac1'
CRS-2673: Attempting to stop 'ora.asm' on 'rac1'
CRS-2677: Stop of 'ora.drivers.acfs' on 'rac1' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.evmd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'rac1' succeeded
CRS-2677: Stop of 'ora.asm' on 'rac1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rac1'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rac1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rac1'
CRS-2677: Stop of 'ora.cssd' on 'rac1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rac1'
CRS-2677: Stop of 'ora.gipcd' on 'rac1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rac1' has completed
CRS-4133: Oracle High Availability Services has been stopped.

[root@rac1 ~]#  /u01/app/12.1.0.2/grid/bin/crsctl  start crs
CRS-4123: Oracle High Availability Services has been started.

[root@rac2 ~]# /u01/app/12.1.0.2/grid/bin/crsctl  start crs
CRS-4123: Oracle High Availability Services has been started.

---
#修復LVM mirror disk的過程
# 最後模擬lvm mirror disk corruption

#原建立LVM mirror disk的方法
#vgcreate vg-crsmirror /dev/sdj /dev/sdk

[root@rac1 ~]# dd if=/dev/zero of=/dev/sdk bs=1M count=512
512+0 records in
512+0 records out
536870912 bytes (537 MB) copied, 2.18916 s, 245 MB/s

[root@rac1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
[root@rac1 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
DISK6
NEWOCR1
NEWOCR2
NEWOCR3

[root@rac2 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
DISK6
NEWOCR1
NEWOCR2
NEWOCR3

#reboot both rac nodes , 檢查RAC狀態無法啟動 , 發現lvmmirror 有問題

[root@rac1 rac1]# pvscan -vP
  PARTIAL MODE. Incomplete logical volumes will be processed.
    Wiping cache of LVM-capable devices
    Wiping internal VG cache
    Walking through all physical volumes
  Couldn't find device with uuid BkdUPV-bDSf-Q0Ps-XSdy-cK47-97pD-7adxaW.
    There are 1 physical volumes missing.
  PV /dev/sdj         VG vg-crsmirror   lvm2 [2.00 GiB / 4.00 MiB free]
  PV unknown device   VG vg-crsmirror   lvm2 [2.00 GiB / 0    free]
  PV /dev/sda2        VG vg_rac1        lvm2 [39.51 GiB / 0    free]
  Total: 3 [43.50 GiB] / in use: 3 [43.50 GiB] / in no VG: 0 [0   ]
[root@rac1 rac1]# vgscan -vP
  PARTIAL MODE. Incomplete logical volumes will be processed.
    Wiping cache of LVM-capable devices
    Wiping internal VG cache
  Reading all physical volumes.  This may take a while...
    Finding all volume groups
    Finding volume group "vg-crsmirror"
  Couldn't find device with uuid BkdUPV-bDSf-Q0Ps-XSdy-cK47-97pD-7adxaW.
    There are 1 physical volumes missing.
    There are 1 physical volumes missing.
  Found volume group "vg-crsmirror" using metadata type lvm2
    Finding volume group "vg_rac1"
  Found volume group "vg_rac1" using metadata type lvm2
[root@rac1 rac1]# ls /etc/lvm/archive
vg-crsmirror_00000-577242756.vg   vg-crsmirror_00010-1015974208.vg
vg-crsmirror_00001-1371052698.vg  vg-crsmirror_00011-454078281.vg
vg-crsmirror_00002-1418557307.vg  vg-crsmirror_00012-354651776.vg
vg-crsmirror_00003-1647225163.vg  vg-crsmirror_00013-1375971973.vg
vg-crsmirror_00004-696018933.vg   vg-crsmirror_00014-1967818712.vg
vg-crsmirror_00005-82168042.vg    vg-crsmirror_00015-64490743.vg
vg-crsmirror_00006-554305478.vg   vg-crsmirror_00016-955890756.vg
vg-crsmirror_00007-1163445813.vg  vg-crsmirror_00017-1706169581.vg
vg-crsmirror_00008-909403487.vg   vg-crsmirror_00018-465009429.vg
vg-crsmirror_00009-1210442952.vg  vg_rac1_00000-913431249.vg
[root@rac1 rac1]#

pvcreate --uuid "BkdUPV-bDSf-Q0Ps-XSdy-cK47-97pD-7adxaW" --restorefile /etc/lvm/archive/vg-crsmirror_00018-465009429.vg /dev/sdk

[root@rac1 rac1]# pvcreate --uuid "BkdUPV-bDSf-Q0Ps-XSdy-cK47-97pD-7adxaW" --restorefile /etc/lvm/archive/vg-crsmirror_00018-465009429.vg /dev/sdk
  Couldn't find device with uuid BkdUPV-bDSf-Q0Ps-XSdy-cK47-97pD-7adxaW.
  Physical volume "/dev/sdk" successfully created

[root@rac1 rac1]# vgcfgrestore vg-crsmirror
  Restored volume group vg-crsmirror

[root@rac1 rac1]# lvs -a -o +devices
  LV                   VG           Attr       LSize   Pool Origin Data%  Meta%  Move Log            Cpy%Sync Convert Devices                                  
  lv_mirror            vg-crsmirror mwi---m---   1.99g                                lv_mirror_mlog                  lv_mirror_mimage_0(0),lv_mirror_mimage_1(0)
  [lv_mirror_mimage_0] vg-crsmirror Iwi---m---   1.99g                                                                /dev/sdj(0)                              
  [lv_mirror_mimage_1] vg-crsmirror Iwi---m---   1.99g                                                                /dev/sdk(0)                              
  [lv_mirror_mlog]     vg-crsmirror lwi---m---   4.00m                                                                /dev/sdk(510)                            
  lv_root              vg_rac1      -wi-ao----  39.01g                                                                /dev/sda2(0)                            
  lv_swap              vg_rac1      -wi-ao---- 512.00m                                                                /dev/sda2(9986)                          

[root@rac1 rac1]#  lvchange -ay /dev/vg-crsmirror/lv_mirror

[root@rac1 rac1]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
DISK6
NEWOCR1
NEWOCR2
[root@rac1 rac1]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
Instantiating disk "NEWOCR3"
[root@rac1 rac1]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
DISK6
NEWOCR1
NEWOCR2
NEWOCR3

[root@rac2 ~]#  lvchange -ay /dev/vg-crsmirror/lv_mirror
[root@rac2 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
Instantiating disk "NEWOCR3"
[root@rac2 ~]# oracleasm listdisks
DISK1
DISK2
DISK3
DISK4
DISK5
DISK6
NEWOCR1
NEWOCR2
NEWOCR3

#Check LVM mirror status
[root@rac1 rac1]# lvs | fgrep "mwi"
  lv_mirror vg-crsmirror mwi-a-m---   1.99g                                lv_mirror_mlog 100.00

[root@rac2 ~]# lvs | fgrep "mwi"
  lv_mirror vg-crsmirror mwi-a-m---   1.99g                                lv_mirror_mlog 100.00


#reboot db1
[oracle@rac1 ~]$ srvctl status database -d orcl
Instance orcl1 is running on node rac1
Instance orcl2 is not running on node rac2

#restart db2 crs daemon
[root@rac2 ~]# /u01/app/12.1.0.2/grid/bin/crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'rac2'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'rac2'
CRS-2673: Attempting to stop 'ora.ctssd' on 'rac2'
CRS-2673: Attempting to stop 'ora.evmd' on 'rac2'
CRS-2673: Attempting to stop 'ora.asm' on 'rac2'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'rac2'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'rac2'
CRS-2677: Stop of 'ora.drivers.acfs' on 'rac2' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'rac2' succeeded
CRS-2677: Stop of 'ora.evmd' on 'rac2' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'rac2' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'rac2' succeeded
CRS-2677: Stop of 'ora.asm' on 'rac2' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'rac2'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'rac2' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'rac2'
CRS-2677: Stop of 'ora.cssd' on 'rac2' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'rac2'
CRS-2677: Stop of 'ora.gipcd' on 'rac2' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'rac2' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@rac2 ~]# /u01/app/12.1.0.2/grid/bin/crsctl  start crs
CRS-4123: Oracle High Availability Services has been started.

[root@rac2 ~]# ps -ef |grep pmon
grid     18563     1  0 03:02 ?        00:00:00 asm_pmon_+ASM2
oracle   21650     1  0 03:03 ?        00:00:00 ora_pmon_orcl2
root     22085 18525  0 03:03 pts/1    00:00:00 grep pmon
[root@rac2 ~]# su - oracle
[oracle@rac2 ~]$ srvctl status database -d orcl
Instance orcl1 is running on node rac1
Instance orcl2 is not running on node rac2
[oracle@rac2 ~]$ srvctl status database -d orcl
Instance orcl1 is running on node rac1
Instance orcl2 is running on node rac2
[oracle@rac2 ~]$

-------------------------------------------------------------------------------



ref:
https://bbs.archlinux.org/viewtopic.php?id=28942
http://www.ewan.cc/?q=node/108
http://unix.stackexchange.com/questions/12452/how-to-check-lvm-mirroring-under-linux

沒有留言:

LinkWithin-相關文件

Related Posts Plugin for WordPress, Blogger...