Monday, December 6, 2010

Using fked to restore ASM Disk Header (test)

Recently i tested kfed for restoring ASM disk header corruption. Here are the steps:


1-) Create a test Diskgroup,  test tablespace and test table
CREATE DISKGROUP TEST EXTERNAL REDUNDANCY
DISK
'/dev/rhdisk6',
'/dev/rhdisk7'
ATTRIBUTE
'compatible.asm' = '11.2',
'compatible.rdbms' = '11.2';


CREATE TABLESPACE TEST_TS DATAFILE '+TEST' SIZE 100M AUTOEXTEND ON NEXT 125M MAXSIZE 10240M
LOGGING
ONLINE
PERMANENT
EXTENT MANAGEMENT LOCAL AUTOALLOCATE
BLOCKSIZE 8K
SEGMENT SPACE MANAGEMENT AUTO
FLASHBACK ON;


create table SCOTT.TEST_TABLE (a int, b varchar2(1000)) tablespace TEST_TS;
insert into SCOTT.TEST_TABLE values (1, 'asddghmsddfuh');
insert into SCOTT.TEST_TABLE values (2, 'asddghmsddfuh');
insert into SCOTT.TEST_TABLE values (3,'sdfghfasdfgywbefg');

2-) Check ASM
SQL> select GROUP_NUMBER, NAME, ALLOCATION_UNIT_SIZE, TYPE, TOTAL_MB-FREE_MB USED_MB, TOTAL_MB, COMPATIBILITY, DATABASE_COMPATIBILITY from v$asm_diskgroup;
GROUP_NUMBER NAME                           ALLOCATION_UNIT_SIZE TYPE      USED_MB   TOTAL_MB COMPATIBIL DATABASE_C
------------ ------------------------------ -------------------- ------ ---------- ---------- ---------- ----------
           1 DATACRS                                     1048576 EXTERN       2123      10240 11.2.0.0.0 10.1.0.0.0
           2 TEST                                        1048576 EXTERN        163      10240 11.2.0.0.0 11.2.0.0.0


SQL> select GROUP_NUMBER, HEADER_STATUS, PATH,NAME, LIBRARY, FAILGROUP_TYPE,  trunc(OS_MB/1024*1024*1024/1024/1024) SUM_OS from V$ASM_DISK;
GROUP_NUMBER HEADER_STATU PATH                           NAME                           LIBRARY              FAILGRO     SUM_OS
------------ ------------ ------------------------------ ------------------------------ -------------------- ------- ----------
           0 CANDIDATE    /dev/rhdisk9                                                  System               REGULAR          5
           0 CANDIDATE    /dev/rhdisk8                                                  System               REGULAR          5
           1 MEMBER       /dev/rhdisk2                   DATACRS_0000                   System               REGULAR          5
           1 MEMBER       /dev/rhdisk3                   DATACRS_0001                   System               REGULAR          5
           2 MEMBER       /dev/rhdisk6                   TEST_0000                      System               REGULAR          5
           2 MEMBER       /dev/rhdisk7                   TEST_0001                      System               REGULAR          5


2-) Corrupt disk hdisk6 (with root):

node1
lquerypv -h /dev/rhdisk6
00000000   00820101 00000000 80000000 B8CC7719  |..............w.|
00000010   00000000 00000000 00000000 00000000  |................|
00000020   4F52434C 4449534B 00000000 00000000  |ORCLDISK........|
00000030   00000000 00000000 00000000 00000000  |................|
00000040   0B200000 00000103 54455354 5F303030  |. ......TEST_000|
00000050   30000000 00000000 00000000 00000000  |0...............|
00000060   00000000 00000000 54455354 00000000  |........TEST....|
00000070   00000000 00000000 00000000 00000000  |................|
00000080   00000000 00000000 54455354 5F303030  |........TEST_000|
00000090   30000000 00000000 00000000 00000000  |0...............|
000000A0   00000000 00000000 00000000 00000000  |................|
000000B0   00000000 00000000 00000000 00000000  |................|
000000C0   00000000 00000000 01F6B0CB 6422B400  |............d"..|
000000D0   01F6B0CB 64C47000 02001000 00100000  |....d.p.........|
000000E0   0001BC80 00001400 00000002 00000001  |................|
000000F0   00000002 00000002 00000000 00000000  |................|

#stop crs before. Other case ASM will protect disk from corruption
./crsctl stop crs  
chdev -l hdisk6 -a pv=yes
lquerypv -h /dev/rhdisk6 #corrupted!!!
00000000   C9C2D4C1 00000000 00000000 00000000  |................|
00000010   00000000 00000000 00000000 00000000  |................|
00000020   00000000 00000000 00000000 00000000  |................|
00000030   00000000 00000000 00000000 00000000  |................|
00000040   00000000 00000000 00000000 00000000  |................|
00000050   00000000 00000000 00000000 00000000  |................|
00000060   00000000 00000000 00000000 00000000  |................|
00000070   00000000 00000000 00000000 00000000  |................|
00000080   00C2E250 BB2CE1C7 00000000 00000000  |...P.,..........|
00000090   00000000 00000000 00000000 00000000  |................|
000000A0   00000000 00000000 00000000 00000000  |................|
000000B0   00000000 00000000 00000000 00000000  |................|
000000C0   00000000 00000000 00000000 00000000  |................|
000000D0   00000000 00000000 00000000 00000000  |................|
000000E0   00000000 00000000 00000000 00000000  |................|
000000F0   00000000 00000000 00000000 00000000  |................|


3-) TEST DG seem to be unmounted. Try to mount it:

ASMCMD> mount TEST
ORA-15032: not all alterations performed
ORA-15017: diskgroup "TEST" cannot be mounted
ORA-15063: ASM discovered an insufficient number of disks for diskgroup "TEST" (DBD ERROR: OCIStmtExecute)


SQL> select * from SCOTT.TEST_TABLE ;
select * from SCOTT.TEST_TABLE
                    *
ERROR at line 1:
ORA-01157: cannot identify/lock data file 6 - see DBWR trace file
ORA-01110: data file 6: '+TEST/mydb/datafile/test_ts.256.737033885'


4-) Restore disk header using kfed (with root)
kfed REPAIR /dev/rhdisk6
#now it seems to be ok!
lquerypv -h /dev/rhdisk6
00000000   00820101 00000000 80000000 B8CC7719  |..............w.|
00000010   00000000 00000000 00000000 00000000  |................|
00000020   4F52434C 4449534B 00000000 00000000  |ORCLDISK........|
00000030   00000000 00000000 00000000 00000000  |................|
00000040   0B200000 00000103 54455354 5F303030  |. ......TEST_000|
00000050   30000000 00000000 00000000 00000000  |0...............|
00000060   00000000 00000000 54455354 00000000  |........TEST....|
00000070   00000000 00000000 00000000 00000000  |................|
00000080   00000000 00000000 54455354 5F303030  |........TEST_000|
00000090   30000000 00000000 00000000 00000000  |0...............|
000000A0   00000000 00000000 00000000 00000000  |................|
000000B0   00000000 00000000 00000000 00000000  |................|
000000C0   00000000 00000000 01F6B0CB 6422B400  |............d"..|
000000D0   01F6B0CB 64C47000 02001000 00100000  |....d.p.........|
000000E0   0001BC80 00001400 00000002 00000001  |................|
000000F0   00000002 00000002 00000000 00000000  |................|

5-) Try to mount again:

ASMCMD> mount TEST #ok!!!

6-) Now do some tests (everything ok)
CREATE TABLESPACE TEST_TS2 DATAFILE '+TEST' SIZE 100M AUTOEXTEND ON NEXT 125M MAXSIZE 10240M
LOGGING
ONLINE
PERMANENT
EXTENT MANAGEMENT LOCAL AUTOALLOCATE
BLOCKSIZE 8K
SEGMENT SPACE MANAGEMENT AUTO
FLASHBACK ON;
create table SCOTT.TEST_TABLE2 (a int, b varchar2(1000)) tablespace TEST_TS;

insert into SCOTT.TEST_TABLE2 values (4, 'asddghmsddfuh');
insert into SCOTT.TEST_TABLE2 values (5, 'asddghmsddfuh');
insert into SCOTT.TEST_TABLE2 values (6,'sdfghfasdfgywbefg');
drop table SCOTT.TEST_TABLE;
drop tablespace TEST_TS;
alter diskgroup TEST add disk '/dev/rhdisk8','/dev/rhdisk9' drop disk 'TEST_0000','TEST_0001';


SQL> select GROUP_NUMBER, HEADER_STATUS, PATH,NAME, LIBRARY, FAILGROUP_TYPE,  trunc(OS_MB/1024*1024*1024/1024/1024) SUM_OS from V$ASM_DISK;
GROUP_NUMBER HEADER_STATU PATH                           NAME                           LIBRARY              FAILGRO     SUM_OS
------------ ------------ ------------------------------ ------------------------------ -------------------- ------- ----------
           0 FORMER       /dev/rhdisk6                                                  System               REGULAR          5
           0 FORMER       /dev/rhdisk7                                                  System               REGULAR          5
           1 MEMBER       /dev/rhdisk2                   DATACRS_0000                   System               REGULAR          5
           1 MEMBER       /dev/rhdisk3                   DATACRS_0001                   System               REGULAR          5
           2 MEMBER       /dev/rhdisk8                   TEST_0002                      System               REGULAR          5
           2 MEMBER       /dev/rhdisk9                   TEST_0003                      System               REGULAR          5

No comments: