星期三, 1月 06, 2016

Oracle 12c RAC VOTING disk破壞測試(normal reundancy)

我的測試環境:
GRID 12.1.0.2
DB 12.1.0.2
使用兩台主機, 分別跑instance1, instance2

#VOTING disk破壞測試(後來發現我的LAB, VOTING disk不是用normal redunancy)
dd if=/dev/zero of=/dev/sdi1 bs=1M count=20  #disk8
dd if=/dev/zero of=/dev/sdj1 bs=1M count=20  #disk9

/u01/app/12.1.0/grid/bin/crsctl stop crs -f
/u01/app/12.1.0/grid/bin/crsctl start crs -excl


SQL> select name , state from v$asm_diskgroup;

NAME                           STATE
------------------------------ -----------
VOTING                         DISMOUNTED
DATA                           MOUNTED

SQL>
col path for a25
select path, MOUNT_STATUS, HEADER_STATUS, MODE_STATUS, STATE from v$asm_disk;

SQL>
PATH                      MOUNT_S HEADER_STATU MODE_ST STATE
------------------------- ------- ------------ ------- --------
ORCL:DISK8                CLOSED  PROVISIONED  ONLINE  NORMAL
ORCL:DISK7                CLOSED  MEMBER       ONLINE  NORMAL
ORCL:DISK9                CLOSED  PROVISIONED  ONLINE  NORMAL
ORCL:DISK1                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK2                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK3                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK4                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK5                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK6                CACHED  MEMBER       ONLINE  NORMAL

V$ASM_DISK不應當顯示PROVISIONED
我們發現當checksum不正確時可能導致V$ASM_DISK.HEADER_STATUS顯示為PROVISIONED

dd if=/dev/zero of=/dev/sdh1 bs=1M count=20

/usr/sbin/oracleasm createdisk disk10 /dev/sdi1
/usr/sbin/oracleasm createdisk disk11 /dev/sdj1
/usr/sbin/oracleasm createdisk disk12 /dev/sdh1 (ORCL:DISK7, 我以為建立名字不重複的disk name就可以)

[root@db1 ~]# ps -ef |grep ASM |grep pmon
grid     21216     1  0 14:05 ?        00:00:00 asm_pmon_+ASM1
[root@db1 ~]#


su - grid
export ORACLE_HOME=/u01/app/12.1.0/grid
sqlplus / as sysasm
SQL>
col path for a25
select path, MOUNT_STATUS, HEADER_STATUS, MODE_STATUS, STATE from v$asm_disk;

PATH                      MOUNT_S HEADER_STATU MODE_ST STATE
------------------------- ------- ------------ ------- --------
ORCL:DISK11               CLOSED  PROVISIONED  ONLINE  NORMAL  ->PROVISIONED代表header資訊都已經被清空
ORCL:DISK12               CLOSED  PROVISIONED  ONLINE  NORMAL  ->PROVISIONED代表header資訊都已經被清空
ORCL:DISK10               CLOSED  PROVISIONED  ONLINE  NORMAL  ->PROVISIONED代表header資訊都已經被清空
ORCL:DISK1                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK2                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK3                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK4                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK5                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK6                CACHED  MEMBER       ONLINE  NORMAL

9 rows selected.

SQL>
col path for a25
col failgroup  for a10
select path, MOUNT_STATUS, HEADER_STATUS, MODE_STATUS, STATE, failgroup from v$asm_disk;

PATH                      MOUNT_S HEADER_STATU MODE_ST STATE
------------------------- ------- ------------ ------- --------
ORCL:DISK11               CLOSED  PROVISIONED  ONLINE  NORMAL
ORCL:DISK12               CLOSED  PROVISIONED  ONLINE  NORMAL
ORCL:DISK10               CLOSED  PROVISIONED  ONLINE  NORMAL
ORCL:DISK1                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK2                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK3                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK4                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK5                CACHED  MEMBER       ONLINE  NORMAL
ORCL:DISK6                CACHED  MEMBER       ONLINE  NORMAL

9 rows selected.


SQL> alter diskgroup all mount;  -->手動mount all diskgroup發現mount不起來
alter diskgroup all mount
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15017: diskgroup "VOTING" cannot be mounted
ORA-15040: diskgroup is incomplete
ORA-15017: diskgroup "DATA" cannot be mounted
ORA-15013: diskgroup "DATA" is already mounted


SQL>

drop diskgroup voting including contents ;  -->也無法drop voting diskgroup

*
ERROR at line 1:
ORA-15039: diskgroup not dropped
ORA-15001: diskgroup "VOTING" does not exist or is not mounted

SQL> SELECT name, failgroup FROM v$asm_disk;

NAME                           FAILGROUP
------------------------------ ------------------------------



DISK1                          DISK1
DISK2                          DISK2
DISK3                          DISK3
DISK4                          DISK4
DISK5                          DISK5
DISK6                          DISK6

9 rows selected.


SQL>
--查了一下MOS上面的資訊, 說是要重新建一個同名的diskgroup , drop diskgroup把資訊清乾淨

CREATE DISKGROUP VOTING NORMAL REDUNDANCY
FAILGROUP FG1 DISK 'ORCL:DISK10'
FAILGROUP FG2 DISK 'ORCL:DISK11' ,'ORCL:DISK12'
ATTRIBUTE 'au_size'='4M',
          'compatible.asm' = '11.2',
          'compatible.rdbms' = '11.2';

SQL> drop diskgroup VOTING;


CREATE DISKGROUP NEWVOTING NORMAL REDUNDANCY
FAILGROUP FG1 DISK 'ORCL:DISK10'
FAILGROUP FG2 DISK 'ORCL:DISK11'
FAILGROUP FG3 DISK 'ORCL:DISK12'
ATTRIBUTE 'au_size'='4M',
          'compatible.asm' = '11.2',
          'compatible.rdbms' = '11.2';

Diskgroup created.

/usr/sbin/oracleasm createdisk disk13 /dev/sdk1

CREATE DISKGROUP VOTING EXTERNAL REDUNDANCY   -->重新建一個external redunancyvoting disk
DISK 'ORCL:DISK13'
ATTRIBUTE 'au_size'='4M',
          'compatible.asm' = '11.2',
          'compatible.rdbms' = '11.2';

As root:

/u01/app/12.1.0/grid/bin/crsctl replace votedisk +VOTING
[root@db1 ~]# /u01/app/12.1.0/grid/bin/crsctl replace votedisk +VOTING
Successful addition of voting disk 72523eaa74914f02bffd7ab3373e6ecb.
Successfully replaced voting disk group with +VOTING.
CRS-4266: Voting file(s) successfully replaced


/u01/app/12.1.0/grid/bin/crsctl query css votedisk

[root@db1 ~]# /u01/app/12.1.0/grid/bin/crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
1. ONLINE   72523eaa74914f02bffd7ab3373e6ecb (ORCL:DISK13) [VOTING]
Located 1 voting disk(s).
[root@db1 ~]#

/u01/app/12.1.0/grid/bin/crsctl replace votedisk +NEWVOTING
[root@db1 ~]# /u01/app/12.1.0/grid/bin/crsctl replace votedisk +NEWVOTING
Successful addition of voting disk 2b3844998d1a4fc3bfc31b71e92117fc.
Successful addition of voting disk 52c64ccac12e4f01bf30708fd3dda186.
Successful addition of voting disk 2bf78e0bff894f4dbf1bf8bebcce2a17.
Successful deletion of voting disk 72523eaa74914f02bffd7ab3373e6ecb.
Successfully replaced voting disk group with +NEWVOTING.
CRS-4266: Voting file(s) successfully replaced

[root@db1 ~]#  /u01/app/12.1.0/grid/bin/crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
1. ONLINE   2b3844998d1a4fc3bfc31b71e92117fc (ORCL:DISK10) [NEWVOTING]
2. ONLINE   52c64ccac12e4f01bf30708fd3dda186 (ORCL:DISK11) [NEWVOTING]
3. ONLINE   2bf78e0bff894f4dbf1bf8bebcce2a17 (ORCL:DISK12) [NEWVOTING]
Located 3 voting disk(s).
[root@db1 ~]#

------------------------------------------------------------------------
在環境都建立好以後(ASM Diskgroup NEWVOTING 建立三個FG Group : FG1, FG2, FG3

以下再重新破壞一次測試

col path for a25
col failgroup  for a10
select path, MOUNT_STATUS, HEADER_STATUS, MODE_STATUS, STATE, failgroup from v$asm_disk;
PATH                      MOUNT_S HEADER_STATU MODE_ST STATE    FAILGROUP
------------------------- ------- ------------ ------- -------- ----------
ORCL:DISK1                CACHED  MEMBER       ONLINE  NORMAL   DISK1
ORCL:DISK10               CACHED  MEMBER       ONLINE  NORMAL   FG1   --> /dev/sdi1
ORCL:DISK11               CACHED  MEMBER       ONLINE  NORMAL   FG2   --> /dev/sdj1
ORCL:DISK12               CACHED  MEMBER       ONLINE  NORMAL   FG3   --> /dev/sdh1
ORCL:DISK2                CACHED  MEMBER       ONLINE  NORMAL   DISK2
ORCL:DISK3                CACHED  MEMBER       ONLINE  NORMAL   DISK3
ORCL:DISK4                CACHED  MEMBER       ONLINE  NORMAL   DISK4
ORCL:DISK5                CACHED  MEMBER       ONLINE  NORMAL   DISK5
ORCL:DISK6                CACHED  MEMBER       ONLINE  NORMAL   DISK6
ORCL:DISK13               CACHED  MEMBER       ONLINE  NORMAL   DISK13

10 rows selected.

[root@db1 ~]# dd if=/dev/zero of=/dev/sdi1 bs=1M count=20
20+0 records in
20+0 records out
20971520 bytes (21 MB) copied, 0.0299748 s, 700 MB/s
[root@db1 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Cleaning disk "DISK10"
Scanning system for ASM disks...
[root@db1 ~]# oracleasm listdisks
DISK1
DISK11
DISK12
DISK13
DISK2
DISK3
DISK4
DISK5
DISK6

重開機db1之後, 可以觀察到資料庫還是正常啟動
[grid@db1 ~]$ srvctl status database -d msg
Instance msg1 is running on node db1
Instance msg2 is running on node db2

dd if=/dev/zero of=/dev/sdj1 bs=1M count=20

重開機db2之後, 發現db2起不來
[grid@db1 ~]$ srvctl status database -d msg
Instance msg1 is running on node db1
Instance msg2 is not running on node db2

[root@db2 ~]# ps -ef |grep pmon
root      7015  6994  0 15:30 pts/0    00:00:00 grep pmon

接下來使用手動的方式, votingNEWVOTING DISKGROUP移動到VOTING DISKGROUP
[root@db1 ~]# /u01/app/12.1.0/grid/bin/crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
1. ONLINE   c368d52024be4ff9bf626e75dea5d8cf (ORCL:DISK13) [VOTING]
Located 1 voting disk(s).
[root@db1 ~]#

然後再度重開機db2, 就發現資料庫已經修好了, done

[grid@db1 ~]$ srvctl status database -d msg
Instance msg1 is running on node db1
Instance msg2 is running on node db2

dd if=/dev/zero of=/dev/sdi1 bs=1M count=20
dd if=/dev/zero of=/dev/sdj1 bs=1M count=20
dd if=/dev/zero of=/dev/sdh1 bs=1M count=20

/usr/sbin/oracleasm scandisks
/usr/sbin/oracleasm createdisk disk10 /dev/sdh1
/usr/sbin/oracleasm createdisk disk11 /dev/sdi1
/usr/sbin/oracleasm createdisk disk12 /dev/sdj1

[root@db1 ~]# su - grid
[grid@db1 ~]$ sqlplus / as sysasm

SQL*Plus: Release 12.1.0.2.0 Production on Wed Jan 6 15:39:32 2016

Copyright (c) 1982, 2014, Oracle.  All rights reserved.


Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.2.0 - 64bit Production
With the Real Application Clusters and Automatic Storage Management options

SQL>


CREATE DISKGROUP NEWVOTING NORMAL REDUNDANCY
FAILGROUP FG1 DISK 'ORCL:DISK10'
FAILGROUP FG2 DISK 'ORCL:DISK11'
FAILGROUP FG3 DISK 'ORCL:DISK12'
ATTRIBUTE 'au_size'='4M',
          'compatible.asm' = '11.2',
          'compatible.rdbms' = '11.2';

[root@db1 u01]# /u01/app/12.1.0/grid/bin/crsctl replace votedisk +NEWVOTING
Successful addition of voting disk a19e49f96ed24f21bf75bfc4ed3e395a.
Successful addition of voting disk 8cd3b1dd3c5c4f51bf9e31dc08760059.
Successful addition of voting disk cc7de87e4ce74f98bfb9463ba63445f1.
Successful deletion of voting disk 5e3c4d6773174fb1bf8344c7d3efa35d.
Successfully replaced voting disk group with +NEWVOTING.
CRS-4266: Voting file(s) successfully replaced

以上修復完成

心得就是, 不論有幾個voting disk , 各個線上的資料庫節點在voting 毀損的情況下, instance不會受影響
受影響的是下一次開機時, asm/ clusterware因無法偵測到足夠的voting disk票數, 就不會讓資料庫開起來

ref:
http://jaychu649.blogspot.tw/2015/11/11g-racocr-voring.html
http://jaychu649.blogspot.tw/2015/12/oracle-11g-rac-normal-redundancy-voting.html

沒有留言:

LinkWithin-相關文件

Related Posts Plugin for WordPress, Blogger...