#################################
# Create datapump directory
#################################
CREATE DIRECTORY datapump as '/u01/datapump/datapump_exports';
GRANT READ,WRITE ON DIRECTORY datapump TO SYSTEM;
###############################################
# Create shell script to pass the scp password
###############################################
#!/usr/bin/expect -f
# connect via scp
spawn scp /u01/datapump/datapump_exports/exp_*.dmp* "oracle@testserver:/u01/datapump/"
expect {
-re ".*es.*o.*" {
exp_send "yes\r"
exp_continue
}
-re ".*sword.*" {
exp_send "Password\r"
}
}
interact
#############################################################
# Create shell script for the entire datapump export process
#############################################################
#!/bin/ksh
# First, we must set the environment . . . .
ORACLE_SID=$1
export ORACLE_SID
ORACLE_HOME=`cat /etc/oratab|grep ^$ORACLE_SID:|cut -f2 -d':'`
export ORACLE_HOME
PATH=$PATH:/u01/datapump/datapump_exports
DIR=/u01/datapump/datapump_exports
LOGDIR=/u01/datapump/datapump_exports
EXPLOGDIR=/u01/datapump/datapump_exports
export PATH DIR LOGDIR EXPLOGDIR
. oraenv
ADMIN=`cat /u01/datapump/datapump_exports/bkup_mail_list.lis`
EM_LIST='henndue14@gmail.com'
today=`date '+%m/%d/%y %H:%M:%S'`
HOSTNAME=`hostname`
expdate=`date '+%d%m%Y'`
user=`cat /u01/datapump/datapump_exports/.pwdapp |grep ^$ORACLE_SID:|cut -f2 -d':'`
LOGIN=`cat /u01/datapump/datapump_exports/.pwdapp |grep ^$ORACLE_SID:|cut -f3 -d':'`
TEMP_FILE=$LOGDIR/expdp_log.txt
#################################
# Removing 7 days old dump files
#################################
echo "Start dmp file cleanup $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
find $EXPLOGDIR/exp_*.dmp* -mtime +7 -exec rm {} \;
echo "Finished dmp file cleanup $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
#################################
# Alter undo_retention parameter
#################################
echo "Start UNDO_RETUENTION Update $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
$ORACLE_HOME/bin/sqlplus -s <<!
/ as sysdba
whenever sqlerror exit 2;
set echo off
set feedback off
spool $LOGDIR/undo.err
alter system set undo_retention=19169 scope=both;
commit;
spool off
!
undo_exit_status=$?
if [ $undo_exit_status -eq 0 ]
then
echo "undo_retention successfully updated"
else
mailx -s "Update undo_retention on Database $ORACLE_SID Failed!!! "henndue14@gmail.com < $LOGDIR/undo.err
exit 1
fi
echo "Finish UNDO_RETUENTION Update $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
#################################
# Export Schema
#################################
echo "Start datapump Export $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
expdp $LOGIN PARALLEL=6 dumpfile=exp_$(date +%Y%m%d)_%U directory=datapump logfile=exp_$(date +%Y%m%d).log schemas=hennco EXCLUDE=TABLE:\"IN \(\'TEST_TBL\'\)\" flashback_time=systimestamp
exp_exit_status=$?
if [ $exp_exit_status -eq 0 ]
then
echo "Export finished Successfully"
else
cat $EXPLOGDIR/exp_$(date +%Y%m%d).log | egrep -i "ORA-|Failing|error" > $LOGDIR/expdp.err
mailx -s "Export on Database $ORACLE_SID Failed!!! " henndue14@gmail.com < $LOGDIR/expdp.err
exit 1
fi
echo "Finish datapump Export $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
#################################
# Alter undo_retention parameter
#################################
echo "Start UNDO_RETUENTION Update $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
$ORACLE_HOME/bin/sqlplus -s <<!
/ as sysdba
whenever sqlerror exit 2;
set echo off
set feedback off
spool $LOGDIR/undo.err
alter system set undo_retention=1800 scope=both;
commit;
spool off
!
undo_exit_status=$?
if [ $undo_exit_status -eq 0 ]
then
echo "undo_retention successfully updated"
else
mailx -s "Update undo_retention on Database $ORACLE_SID Failed!!! " henndue14@gmail.com < $LOGDIR/undo.err
exit 1
fi
echo "Finish UNDO_RETUENTION Update $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
#################################
# Grant permissions for dmp files
#################################
echo "Start dmp file permission Update $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
chmod -R 775 $EXPLOGDIR/exp_*.dmp
echo "Finish dmp file permission Update $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
#################################
# Compressing files
#################################
echo "Start compressing dmp files $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
gzip $EXPLOGDIR/exp_*.dmp
echo "Finish compressing dmp files $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
#################################
# SCP dmp files to lihbg000tpcdu
#################################
echo "Start dmp file scp $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
/u01/datapump/datapump_exports/scp_dump.sh
echo "Finish dmp file scp $(date)" >> $TEMP_FILE
echo "\n" >> $TEMP_FILE
#################################
# Email export process output
#################################
mailx -s "Database Export process has successfully completed" henndue14@gmail.com < $TEMP_FILE
#################################
# Log file cleanup
#################################
rm /u01/datapump/datapump_exports/expdp_log.txt
rm /u01/datapump/datapump_exports/undo.err
rm /u01/datapump/datapump_exports/expdp.err
exit;
Monday, August 31, 2015
Friday, July 17, 2015
How To Configure SSH for a RAC Installation (Doc ID 300548.1)
To configure SSH you need to perform the following steps on each node in the cluster.
$ cd $HOME
$ mkdir .ssh
$ chmod 700 .ssh
$ cd .ssh
$ ssh-keygen -t rsa
Now accept the default location for the key file
Enter and confirm a passphrase. (you can also press enter twice).
$ ssh-keygen -t dsa
Now accept the default location for the key file
Enter and confirm a passphrase. (you can also press enter twice).
$ cat *.pub >> authorized_keys.<nodeX> (nodeX could be the nodename to differentiate files later)
Now do the same steps on the other nodes in the cluster.
When all those steps are done on the other nodes, start to copy the authorized_keys.<nodeX> to all the nodes into $HOME/.ssh/
For example if you have 4 nodes you will have after the copy in the .ssh 4 files with the name authorized_keys.<nodeX>
Then on EACH node continue the configuration of SSH by doing the following:
$ cd $HOME/.ssh
$ cat *.node* >> authorized_keys
$ chmod 600 authorized_keys
NOTE: ALL public keys must appear in ALL authorized_keys files, INCLUDING the LOCAL public key for each node.
To test that everything is working correct now execute the commands
$ ssh <hostnameX> date
So on example in a 4 node environment:
$ ssh node1 date
$ ssh node2 date
$ ssh node3 date
$ ssh node4 date
Repeat this 4 times on each node, including ssh back to the node itself. The nodeX is the hostname of the node.
The first time you will be asked to add the node to a file called 'known_hosts' this is correct and answer the question with 'yes'. After that when correctly configured you must be able to get the date returned and you will not be prompted for a password.
Note: the above will work if during RSA and DSA configuration no password was provided. If you provide a password then you need to do 2 addition steps.
$ exec /usr/bin/ssh-agent $SHELL
$ /usr/bin/ssh-add
These statements will inform the ssh agent to add the keys to the shell used. So when a new shell is started you need to repeat the last to statements to make sure ssh is working properly.
Please note, ssh will not allow passwordless access if permissions on the home directory of the account you are using allow write access for everyone.
You will also see permission denied error when the permissions on $HOME are 777 or 775. $HOME should have permission 700 or 755.
$ cd $HOME
$ mkdir .ssh
$ chmod 700 .ssh
$ cd .ssh
$ ssh-keygen -t rsa
Now accept the default location for the key file
Enter and confirm a passphrase. (you can also press enter twice).
$ ssh-keygen -t dsa
Now accept the default location for the key file
Enter and confirm a passphrase. (you can also press enter twice).
$ cat *.pub >> authorized_keys.<nodeX> (nodeX could be the nodename to differentiate files later)
Now do the same steps on the other nodes in the cluster.
When all those steps are done on the other nodes, start to copy the authorized_keys.<nodeX> to all the nodes into $HOME/.ssh/
For example if you have 4 nodes you will have after the copy in the .ssh 4 files with the name authorized_keys.<nodeX>
Then on EACH node continue the configuration of SSH by doing the following:
$ cd $HOME/.ssh
$ cat *.node* >> authorized_keys
$ chmod 600 authorized_keys
NOTE: ALL public keys must appear in ALL authorized_keys files, INCLUDING the LOCAL public key for each node.
To test that everything is working correct now execute the commands
$ ssh <hostnameX> date
So on example in a 4 node environment:
$ ssh node1 date
$ ssh node2 date
$ ssh node3 date
$ ssh node4 date
Repeat this 4 times on each node, including ssh back to the node itself. The nodeX is the hostname of the node.
The first time you will be asked to add the node to a file called 'known_hosts' this is correct and answer the question with 'yes'. After that when correctly configured you must be able to get the date returned and you will not be prompted for a password.
Note: the above will work if during RSA and DSA configuration no password was provided. If you provide a password then you need to do 2 addition steps.
$ exec /usr/bin/ssh-agent $SHELL
$ /usr/bin/ssh-add
These statements will inform the ssh agent to add the keys to the shell used. So when a new shell is started you need to repeat the last to statements to make sure ssh is working properly.
Please note, ssh will not allow passwordless access if permissions on the home directory of the account you are using allow write access for everyone.
You will also see permission denied error when the permissions on $HOME are 777 or 775. $HOME should have permission 700 or 755.
How To Download And Install The Latest OPatch Version (Doc ID 274526.1)
1) Please download the latest OPatch version from My Oracle Support (MOS) using the next reference:
Patch 6880880
Or using the next URL:
https://updates.oracle.com/download/6880880.html
2) In the "Platform" field select the relevant platform that corresponds to the Oracle product.
3) Click the Download button
2) Upload the opatch.zip file to your OS host and unzip it:
Example:
3)You can launch OPatch tool by using the absolute path :
ex.:
Or you can set the OPatch directory in your PATH variable so you can execute the OPatch command (opatch <args>) from anywhere without using the absolute path.
ex.:
Patch 6880880
Or using the next URL:
https://updates.oracle.com/download/6880880.html
2) In the "Platform" field select the relevant platform that corresponds to the Oracle product.
3) Click the Download button
2) Upload the opatch.zip file to your OS host and unzip it:
Example:
% mv p6880880_101000_SOLARIS64.zip $ORACLE_HOME
% cd $ORACLE_HOME
(If a former OPatch directory already exist, then rename it)
% mv Opatch OPatch_100057_OLD
% unzip p6880880_101000_SOLARIS64.zip
Archive: p6880880_101000_SOLARIS64.zip
creating: OPatch/
creating: OPatch/docs/
inflating: OPatch/docs/FAQ
inflating: OPatch/docs/README.txt
inflating: OPatch/docs/Users_Guide.txt
inflating: OPatch/docs/bt1.txt
inflating: OPatch/docs/bt2.txt
inflating: OPatch/docs/tmp
inflating: OPatch/emdpatch.pl
creating: OPatch/jlib/
inflating: OPatch/jlib/opatch.jar
inflating: OPatch/opatch
inflating: OPatch/opatch.bat
inflating: OPatch/opatch.pl
creating: OPatch/perl_modules/
inflating: OPatch/perl_modules/Apply.pm
inflating: OPatch/perl_modules/AttachHome.pm
inflating: OPatch/perl_modules/Command.pm
inflating: OPatch/perl_modules/LsInventory.pm
inflating: OPatch/perl_modules/Query.pm
inflating: OPatch/perl_modules/RollBack.pm
inflating: OPatch/perl_modules/Version.pm
inflating: OPatch/perl_modules/XML.pm
inflating: OPatch/perl_modules/opatchIO.pm
inflating: OPatch/README.txt
creating: OPatch/ocm/
extracting: OPatch/ocm/ocm.zip
creating: OPatch/ocm/doc/
inflating: OPatch/ocm/doc/license.txt
creating: OPatch/ocm/lib/
inflating: OPatch/ocm/lib/emocmutl.jar
creating: OPatch/ocm/bin/
inflating: OPatch/ocm/bin/emocmrsp
% cd OPatch
% opatch version
Oracle Interim Patch Installer version 1.0.0.0.58
Copyright (c) 2007 Oracle Corporation. All Rights Reserved..
We recommend you refer to the OPatch documentation under
OPatch/docs for usage reference. We also recommend using
the latest OPatch version. For the latest OPatch version
and other support related issues, please refer to document
293369.1 which is viewable from metalink.oracle.com
OPatch Version: 1.0.0.0.58
% cd $ORACLE_HOME
(If a former OPatch directory already exist, then rename it)
% mv Opatch OPatch_100057_OLD
% unzip p6880880_101000_SOLARIS64.zip
Archive: p6880880_101000_SOLARIS64.zip
creating: OPatch/
creating: OPatch/docs/
inflating: OPatch/docs/FAQ
inflating: OPatch/docs/README.txt
inflating: OPatch/docs/Users_Guide.txt
inflating: OPatch/docs/bt1.txt
inflating: OPatch/docs/bt2.txt
inflating: OPatch/docs/tmp
inflating: OPatch/emdpatch.pl
creating: OPatch/jlib/
inflating: OPatch/jlib/opatch.jar
inflating: OPatch/opatch
inflating: OPatch/opatch.bat
inflating: OPatch/opatch.pl
creating: OPatch/perl_modules/
inflating: OPatch/perl_modules/Apply.pm
inflating: OPatch/perl_modules/AttachHome.pm
inflating: OPatch/perl_modules/Command.pm
inflating: OPatch/perl_modules/LsInventory.pm
inflating: OPatch/perl_modules/Query.pm
inflating: OPatch/perl_modules/RollBack.pm
inflating: OPatch/perl_modules/Version.pm
inflating: OPatch/perl_modules/XML.pm
inflating: OPatch/perl_modules/opatchIO.pm
inflating: OPatch/README.txt
creating: OPatch/ocm/
extracting: OPatch/ocm/ocm.zip
creating: OPatch/ocm/doc/
inflating: OPatch/ocm/doc/license.txt
creating: OPatch/ocm/lib/
inflating: OPatch/ocm/lib/emocmutl.jar
creating: OPatch/ocm/bin/
inflating: OPatch/ocm/bin/emocmrsp
% cd OPatch
% opatch version
Oracle Interim Patch Installer version 1.0.0.0.58
Copyright (c) 2007 Oracle Corporation. All Rights Reserved..
We recommend you refer to the OPatch documentation under
OPatch/docs for usage reference. We also recommend using
the latest OPatch version. For the latest OPatch version
and other support related issues, please refer to document
293369.1 which is viewable from metalink.oracle.com
OPatch Version: 1.0.0.0.58
3)You can launch OPatch tool by using the absolute path :
ex.:
$ORACLE_HOME/OPatch/opatch version
Or you can set the OPatch directory in your PATH variable so you can execute the OPatch command (opatch <args>) from anywhere without using the absolute path.
ex.:
For Korn / Bourne shell:
% export PATH=$PATH:$ORACLE_HOME/OPatch
For C Shell:
% setenv PATH $PATH:$ORACLE_HOME/OPatch
% export PATH=$PATH:$ORACLE_HOME/OPatch
For C Shell:
% setenv PATH $PATH:$ORACLE_HOME/OPatch
How to Add a RAC Node to an Oracle Cluster
A. Verify Prequisites:
Check System Hardware requirements; make sure it is matching with existing nodes (Recommended).
RAM size (Min 2.5 GB, to match with other nodes 36 GB):
/usr/sbin/lsattr -E -l sys0 -a realmem
Swap Space (Min 16 GB, to match with other nodes 24 GB):
/usr/sbin/lsps –s
Temp directory space (Min 1GB, to match with other nodes 4GB):
/usr/bin/df -k /tmp
Compare the size of /u01 mount point with existing nodes to match the size of software installation location
df –g
1. Create Groups and users:
Make sure the following User and Groups exists.
User: Oracle Groups: Oinstall, DBA
Run the below command in racattack18u:
$id oracle
If they don’t exist than create them.
$mkgroup id=’267’ adms='root' oinstall
$ mkgroup id=’264’ adms='root' dba
$ mkuser id=’501’ pgrp='oinstall' groups='dba' home='/u01' oracle
Id’s matching with current production id’s.
2. Create required Directories
ORACLE_BASE
mkdir –p /u01/app/oracle
chown oracle:oinstall /u01/app/oracle
chmod –R 775 /u01/app/oracle
ORACLE_HOME
mkdir –p /u01/products/rdbms_11203
chown oracle:oinstall /u01/products/rdbms_11203
chmod –R 775 /u01/products/rdbms_11203
GRID_HOME
mkdir –p /u01/products/grid_11203
chown root:oinstall /u01/products/grid_11203
chmod –R 775 /u01/products/grid_11203
3. UDP and TCP Kernal Parameters:
Contact AIX team and request them to verify if the kernel parameters are setup as recommended by Oracle.
http://docs.oracle.com/cd/E11882_01/install.112/e48294/preaix.htm#CWAIX418
4. Check the network requirements
Contact Network team and request to setup the network as recommended by Oracle for cluster environment.
http://docs.oracle.com/cd/E11882_01/install.112/e48294/preaix.htm#CWAIX196
A Oracle database cluster node must have the below ip addresses:
Public ip address
Private ip address
Virtual ip address
Single client access name
Current configuration in racattack16u and racattack17u (Can be found in /etc/hosts file):
Public ip address:
10.10.10.148 racattack16u
10.10.10.150 racattack17u
Private id address:
10.10.10.71 racattack16u
10.10.10.72 racattack17u
Virtual ip address:
10.10.10.149 racora3.gmail.com racora3 # ORA Virt. Address A
10.10.10.151 racora4.gmail.com racora4 # ORA Virt. Address B
Scan IP address:
SCAN IP Addresses for RAC Cluster (addresses selected round robin in DNS):
# 10.10.10.147 \
# 10.10.10.54 } rac-scan
# 10.10.10.155 /
For Current production cluster setup the Single client access name is defined as “rac-scan” , new node racattack18u should be included in it.
Once the configuration is done, run the below command to confirm.
$nslookup rac-scan
Output should be similar in all the 3 nodes.
Current output:oracle@racattack17u:rac2:/u01 $ nslookup rac-scan
Server: 10.10.10.37
Address: 10.10.10.53
Name: rac-scan.li-sec.state.pa.us
Address: 10.10.10.155
Name: rac-scan.li-sec.state.pa.us
Address: 10.10.10.154
Name: rac-scan.li-sec.state.pa.us
Address: 10.10.10.147
5. Make sure that all the ASM disks can be accessed by the Oracle user on new node.
Contact Storage team and request them to present all the storage devices currently used by ASM on production cluster database to racattack18u, and set their ownership to oracle user.
Once done, DBA can verify the same using below steps:
On racattack16u:
$export ORACLE_SID=ASM
$ . oraenv
$sqlplus ‘/as sysasm’
Sql>select path from v$asm_disk;
/dev/asm_128G_P0_1
/dev/asm_128G_P0_2
/dev/asm_128G_P0_3
/dev/asm_128G_P10_1
/dev/asm_128G_P1_1
/dev/asm_128G_P1_2
/dev/asm_128G_P1_3
Total 39 disks available right now…..
Exit out of sqlplus,
$ls –ltr /dev/asm_*
Now connect to racattack18u and run the below command:
$ls –ltr /dev/asm_*
Make sure the result is same for “ls” command in racattack16u and racattack18u.
6. Configure ssh and set user equivalency, Make sure you are able to connect to oracle user of racattack16u, racattack17u from racattack18u using ssh without providing password.
As Oracle user Run the Below commands on racattack18u:
$ cd /u01/.ssh
$mv authorized_keys2 authorized_keys2.old
Now As Oracle user run the below Commands on racattack16u
$cd /u01/.ssh
$scp authorized_keys2 oracle@racattack18u: /u01/.ssh/
$ssh racattack18u
(You should be able to connect without prompting a password)
NOTE: In case something is missed in prerequisites, it can be detected automatically when we run “cluvfy” utility in below steps. Which can be fixed subsequently.
B. Verify using Cluster verification utility for Node add procedure:
1. From an existing node(racattack16u) perform post hardware and OS check
$ cluvfy stage -post hwos -n racattack18u
2. From an existing node run the below command to Compare one of the existing node with new node
$ cluvfy comp peer -refnode racattack16u -n racattack18u -orainv oinstall -osdba dba –verbose
3. Run the below command to Verify the integrity of the cluster and the new node
$cluvfy stage –pre nodeadd –n racattack18u –verbose
We may receive PRVF-5449 Error, it’s an Oracle BUG, workaround is to set the below environment variable.
export IGNORE_PREADDNODE_CHECKS=Y
C. Add Grid Home to the new Node:
Run the below addnode.sh command from racattack16u to add new node to the cluster, this will run Oracle universal installer in silent mode and copy the Oracle Grid_Infrastructure binaries from existing node to new node.
$GRID_HOME/oui/bin/addnode.sh “CLUSTER_NEW_NODES=
{ racattack18u }” “CLUSTER_NEW_VIRTUAL_HOSTNAMES={ racora5}”
At the end of the script, it will prompt to run scripts as root user (in the new node racattack18u), connect to a new session with root user and run the scripts suggested. Once the root scripts are executed successfully, hit enter on the current screen to complete the GRID_HOME installation in new node.
If this is successful than we can see the new node as part of cluster and along with this it starts clusterware deamons, ASM instance, Listener, VIP, Scan services in the new node.
Run the below commands to verify,
$ps –ef|grep grid|grep –v grep (In New Node racattack18u)
$ps –ef|grep d.bin (In New Node racattack18u)
$crsctl check crs (In New Node racattack18u)
$crsctl stat res –t (From any node)
$olsnodes (From any node)
D. Add Oracle Home to the new Node:
Run the below addnode.sh command from racattack16u to add new node to the cluster, this will run Oracle universal installer in silent mode and copy the Oracle rdbms binaries from existing node to new node.
$ORACLE_HOME/oui/bin/addnode.sh “CLUSTER_NEW_NODES=
{ racattack18u }”
At the end of the script, it will prompt to run scripts as root user (in the new node racattack18u), connect to a new session with root user and run the scripts suggested. Hit enter on the current screen to complete the ORACLE_HOME installation in new node.
E. Add instance to the database for newly added node
Invoke DBCA from one of the old node and add new instance to the database.
RAC database-->Instance management-->Add Instance
Do this process for all the databases running under the cluster.
F. Verification of Node addition:
Run the below commands from one of the node racattack16u to confirm the new node and instance are added successfully.
$ cluvfy stage –post nodeadd –n racattack18u -verbose
$ olsnodes
$ crsctl stat res –t
Connect to the new node racattack18u and run the below commands to check the details of newly added node.
$ crsctl check crs
$ ps –ef|grep d.bin
$ ps –ef|grep pmon
Go through the directories of ORACLE_HOME, ORACLE_BASE and GRID_HOME.
Connect to Oracle database instance using sqlplus.
Connect to the ASM instance using asmcmd and sqlplus.
Check System Hardware requirements; make sure it is matching with existing nodes (Recommended).
RAM size (Min 2.5 GB, to match with other nodes 36 GB):
/usr/sbin/lsattr -E -l sys0 -a realmem
Swap Space (Min 16 GB, to match with other nodes 24 GB):
/usr/sbin/lsps –s
Temp directory space (Min 1GB, to match with other nodes 4GB):
/usr/bin/df -k /tmp
Compare the size of /u01 mount point with existing nodes to match the size of software installation location
df –g
1. Create Groups and users:
Make sure the following User and Groups exists.
User: Oracle Groups: Oinstall, DBA
Run the below command in racattack18u:
$id oracle
If they don’t exist than create them.
$mkgroup id=’267’ adms='root' oinstall
$ mkgroup id=’264’ adms='root' dba
$ mkuser id=’501’ pgrp='oinstall' groups='dba' home='/u01' oracle
Id’s matching with current production id’s.
2. Create required Directories
ORACLE_BASE
mkdir –p /u01/app/oracle
chown oracle:oinstall /u01/app/oracle
chmod –R 775 /u01/app/oracle
ORACLE_HOME
mkdir –p /u01/products/rdbms_11203
chown oracle:oinstall /u01/products/rdbms_11203
chmod –R 775 /u01/products/rdbms_11203
GRID_HOME
mkdir –p /u01/products/grid_11203
chown root:oinstall /u01/products/grid_11203
chmod –R 775 /u01/products/grid_11203
3. UDP and TCP Kernal Parameters:
Contact AIX team and request them to verify if the kernel parameters are setup as recommended by Oracle.
http://docs.oracle.com/cd/E11882_01/install.112/e48294/preaix.htm#CWAIX418
4. Check the network requirements
Contact Network team and request to setup the network as recommended by Oracle for cluster environment.
http://docs.oracle.com/cd/E11882_01/install.112/e48294/preaix.htm#CWAIX196
A Oracle database cluster node must have the below ip addresses:
Public ip address
Private ip address
Virtual ip address
Single client access name
Current configuration in racattack16u and racattack17u (Can be found in /etc/hosts file):
Public ip address:
10.10.10.148 racattack16u
10.10.10.150 racattack17u
Private id address:
10.10.10.71 racattack16u
10.10.10.72 racattack17u
Virtual ip address:
10.10.10.149 racora3.gmail.com racora3 # ORA Virt. Address A
10.10.10.151 racora4.gmail.com racora4 # ORA Virt. Address B
Scan IP address:
SCAN IP Addresses for RAC Cluster (addresses selected round robin in DNS):
# 10.10.10.147 \
# 10.10.10.54 } rac-scan
# 10.10.10.155 /
For Current production cluster setup the Single client access name is defined as “rac-scan” , new node racattack18u should be included in it.
Once the configuration is done, run the below command to confirm.
$nslookup rac-scan
Output should be similar in all the 3 nodes.
Current output:oracle@racattack17u:rac2:/u01 $ nslookup rac-scan
Server: 10.10.10.37
Address: 10.10.10.53
Name: rac-scan.li-sec.state.pa.us
Address: 10.10.10.155
Name: rac-scan.li-sec.state.pa.us
Address: 10.10.10.154
Name: rac-scan.li-sec.state.pa.us
Address: 10.10.10.147
5. Make sure that all the ASM disks can be accessed by the Oracle user on new node.
Contact Storage team and request them to present all the storage devices currently used by ASM on production cluster database to racattack18u, and set their ownership to oracle user.
Once done, DBA can verify the same using below steps:
On racattack16u:
$export ORACLE_SID=ASM
$ . oraenv
$sqlplus ‘/as sysasm’
Sql>select path from v$asm_disk;
/dev/asm_128G_P0_1
/dev/asm_128G_P0_2
/dev/asm_128G_P0_3
/dev/asm_128G_P10_1
/dev/asm_128G_P1_1
/dev/asm_128G_P1_2
/dev/asm_128G_P1_3
Total 39 disks available right now…..
Exit out of sqlplus,
$ls –ltr /dev/asm_*
Now connect to racattack18u and run the below command:
$ls –ltr /dev/asm_*
Make sure the result is same for “ls” command in racattack16u and racattack18u.
6. Configure ssh and set user equivalency, Make sure you are able to connect to oracle user of racattack16u, racattack17u from racattack18u using ssh without providing password.
As Oracle user Run the Below commands on racattack18u:
$ cd /u01/.ssh
$mv authorized_keys2 authorized_keys2.old
Now As Oracle user run the below Commands on racattack16u
$cd /u01/.ssh
$scp authorized_keys2 oracle@racattack18u: /u01/.ssh/
$ssh racattack18u
(You should be able to connect without prompting a password)
NOTE: In case something is missed in prerequisites, it can be detected automatically when we run “cluvfy” utility in below steps. Which can be fixed subsequently.
B. Verify using Cluster verification utility for Node add procedure:
1. From an existing node(racattack16u) perform post hardware and OS check
$ cluvfy stage -post hwos -n racattack18u
2. From an existing node run the below command to Compare one of the existing node with new node
$ cluvfy comp peer -refnode racattack16u -n racattack18u -orainv oinstall -osdba dba –verbose
3. Run the below command to Verify the integrity of the cluster and the new node
$cluvfy stage –pre nodeadd –n racattack18u –verbose
We may receive PRVF-5449 Error, it’s an Oracle BUG, workaround is to set the below environment variable.
export IGNORE_PREADDNODE_CHECKS=Y
C. Add Grid Home to the new Node:
Run the below addnode.sh command from racattack16u to add new node to the cluster, this will run Oracle universal installer in silent mode and copy the Oracle Grid_Infrastructure binaries from existing node to new node.
$GRID_HOME/oui/bin/addnode.sh “CLUSTER_NEW_NODES=
{ racattack18u }” “CLUSTER_NEW_VIRTUAL_HOSTNAMES={ racora5}”
At the end of the script, it will prompt to run scripts as root user (in the new node racattack18u), connect to a new session with root user and run the scripts suggested. Once the root scripts are executed successfully, hit enter on the current screen to complete the GRID_HOME installation in new node.
If this is successful than we can see the new node as part of cluster and along with this it starts clusterware deamons, ASM instance, Listener, VIP, Scan services in the new node.
Run the below commands to verify,
$ps –ef|grep grid|grep –v grep (In New Node racattack18u)
$ps –ef|grep d.bin (In New Node racattack18u)
$crsctl check crs (In New Node racattack18u)
$crsctl stat res –t (From any node)
$olsnodes (From any node)
D. Add Oracle Home to the new Node:
Run the below addnode.sh command from racattack16u to add new node to the cluster, this will run Oracle universal installer in silent mode and copy the Oracle rdbms binaries from existing node to new node.
$ORACLE_HOME/oui/bin/addnode.sh “CLUSTER_NEW_NODES=
{ racattack18u }”
At the end of the script, it will prompt to run scripts as root user (in the new node racattack18u), connect to a new session with root user and run the scripts suggested. Hit enter on the current screen to complete the ORACLE_HOME installation in new node.
E. Add instance to the database for newly added node
Invoke DBCA from one of the old node and add new instance to the database.
RAC database-->Instance management-->Add Instance
Do this process for all the databases running under the cluster.
F. Verification of Node addition:
Run the below commands from one of the node racattack16u to confirm the new node and instance are added successfully.
$ cluvfy stage –post nodeadd –n racattack18u -verbose
$ olsnodes
$ crsctl stat res –t
Connect to the new node racattack18u and run the below commands to check the details of newly added node.
$ crsctl check crs
$ ps –ef|grep d.bin
$ ps –ef|grep pmon
Go through the directories of ORACLE_HOME, ORACLE_BASE and GRID_HOME.
Connect to Oracle database instance using sqlplus.
Connect to the ASM instance using asmcmd and sqlplus.
How to remove a RAC node from and Cluster
The following steps the are required to successfully remove a Cluster Node from an existing Cluster:
1) Before starting the node removal process it is always recommended to take backup of OCR and databases.
-Login as root on racattack15u and verify the last backup of OCR
#$GRID_HOME/bin/ocrconfig -showbackup
-If required take a manual backup of OCR
#$GRID_HOME/bin/ocrconfig –manualbackup
2) Unpin the node to be removed if it is pinned
Run the below command from one of the node to check if the node to be removed is pinned or unpinned.
$ olsnodes -t -s
racattack15u Active Unpinned
racattack18u Active Unpinned
(NOTE: Verified they are already unpinned)
-Run the below command if racattack18u is pinned
$crsctl unpin css -n racattack18u
3) Remove the database instance using DBCA
Note: This can also be done using a AIX GUI interface like CYGWIN.
Remove the database instance using DBCA:
-Invoke DBCA from racattack15u as oracle user
$$ORACLE_HOME/bin/dbca
Select the following in each screen to remove all the database instances running in racattack18u
RAC database-->Instance management-->Delete Instance
Repeat to remove all the 3 Instances.
4) Verify the instance has been removed with the following commands on racattack18u.
$srvctl config database -d racdb1
$srvctl config database -d racdb2
$srvctl config database -d racdb3
$ps –ef|grep pmon
$crsctl stat res -t
NOTE: At this point database instances and its related services are removed from cluster. ASM and other clusterware compontents will be removed as part of GRID infrastructure removal.
5) Disable and Stop Listener on the Node to be removed.
oracle@racattack18u::/u01 $ ps -ef|grep tns (To get the running listener_alias name)oracle@racattack18u::/u01 $ lsnrctl status <Listener Alias>oracle@racattack18u::/u01 $ srvctl config listener -a
-Use the below commands to disable the listener(s) which are running from ORACLE_HOMEoracle@racattack18u::/u01 $ srvctl disable listener -l <Listener Name> -n racattack18uoracle@racattack18u::/u01 $ srvctl stop listener -l <Listener Name> -n racattack18u
6) Update Inventory on the Node to be deleted
-Verify the inventory in racattack18u before updating it.
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
-Run the following command on racattack18u to update the inventory
$ /u01/products/rdbms_11203/oui/bin/runInstaller -updateNodeList ORACLE_HOME=/u01/products/rdbms_11203 "CLUSTER_NODES={racattack18u}" -local
-Verify the inventory in racattack18u it should not have information about racattack15u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
7.) Remove the RAC Database Software
The next step is to remove the RAC Database Software from the node to be deleted (racattack18u).
-Run below command as oracle user from racattack18u
$ cd $ORACLE_HOME/deinstall
$ ./deinstall -local
(-local option is to remoce ORACLE_HOME from local node)
8.) Update Inventory on the Remaining Nodes
-Verify the inventory in racattack15u before updating it.
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
-Run the following command on racattack15u to update the inventory
$ cd /u01/products/rdbms_11203/oui/bin
$ runInstaller -updateNodeList ORACLE_HOME=/u01/products/rdbms_11203 "CLUSTER_NODES={racattack15u}" -local
-Verify the inventory in racattack15u it should not have information about racattack18u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
9.) Remove Clusterware
The next step is to remove the clusterware from the node you want to delete.
Note: This must do as root.
-As root on racattack18u run the below command to deconfig clusterware
# cd /u01/products/grid_11203
# pwd
/u01/products/grid_11203
# cd crs/install
# pwd
/u01/products/grid_11203/crs/install
# hostname
racattack18u
# ./rootcrs.pl -deconfig -force
(At this point All the clusterware services including ASM are stopped)
10.) Delete node from Clusterware configuration
The next step is to remove the desired node from the clusterware configuration. This is done with the following commands.
# cd /u01/products/grid_11203/bin
# olsnodes –t –s (For verification)
-As root on racattack15u run the below command to delete racattack18u from cluster
# crsctl delete node -n racattack18u
(racattack18u will be removed from clusterware and registry)
-Run the below command from racattack15u to verify the existing nodes
# olsnodes –t –s (For verification)
(Should only show racattack15u)
11.) Update Inventory on node to be removed as "Grid"
The next step is to update the inventory on the desired node to be removed as the "Grid" software owner (Oracle for this environment).
Note: make sure that is done as the "Grid Owner" which is "Oracle" in this environment.
Verify the inventory in racattack18u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
-Run the following command on racattack18u to update the inventory
$ cd /u01/products/grid_11203/oui/bin
$ runInstaller -updateNodeList ORACLE_HOME=/u01/products/grid_11203 "CLUSTER_NODES={racattack18u}" -local
-Verify the inventory in racattack18u it should not have information about racattack15u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
12.) Remove the Clusterware Software from node to be removed.
The next step is to remove the clusterware software from the desired node (racattack18u).
Note: make sure that is done as the "Grid Owner". which is "Oracle" in this environment.
-Run below command as oracle user from racattack18u
$ cd /u01/products/grid_11203/deinstall
$ ./deinstall -local
(-local option is to remoce ORACLE_HOME from local node)
-In the end it asks us to run a script from a different session as root, run the script
13.) Update Inventory on remaining nodes for "Grid"
The next step is to update inventory on the remaining nodes (racattack15u) for "Grid".
Note: make sure that is done as the "Grid Owner"
Verify the inventory in racattack15u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
- Run the following command on racattack15u to update the inventory
$ cd /u01/products/grid_11203/oui/bin
$ ./runInstaller -updateNodeList ORACLE_HOME=/u01/products/grid_11203 "CLUSTER_NODES={racattack15u}"
-Verify the inventory in racattack15u it should not have information about racattack18u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
14.) Confirm the node has been successfully removed.
The next step is to verify that the node has been successfully removed. To do this, execute the following command.
-Run the below command from racattack15u as oracle useroracle@racattack15u:racdb1:/u01 $ cluvfy stage -post nodedel -n racattack18uoracle@racattack15u:racdb1:/u01 $ crsctl stat res -t
NOTE: At this Stage racattack18u is clean without any Oracle software installations. racattack15u still remains as a single instance cluster database.
1) Before starting the node removal process it is always recommended to take backup of OCR and databases.
-Login as root on racattack15u and verify the last backup of OCR
#$GRID_HOME/bin/ocrconfig -showbackup
-If required take a manual backup of OCR
#$GRID_HOME/bin/ocrconfig –manualbackup
2) Unpin the node to be removed if it is pinned
Run the below command from one of the node to check if the node to be removed is pinned or unpinned.
$ olsnodes -t -s
racattack15u Active Unpinned
racattack18u Active Unpinned
(NOTE: Verified they are already unpinned)
-Run the below command if racattack18u is pinned
$crsctl unpin css -n racattack18u
3) Remove the database instance using DBCA
Note: This can also be done using a AIX GUI interface like CYGWIN.
Remove the database instance using DBCA:
-Invoke DBCA from racattack15u as oracle user
$$ORACLE_HOME/bin/dbca
Select the following in each screen to remove all the database instances running in racattack18u
RAC database-->Instance management-->Delete Instance
Repeat to remove all the 3 Instances.
4) Verify the instance has been removed with the following commands on racattack18u.
$srvctl config database -d racdb1
$srvctl config database -d racdb2
$srvctl config database -d racdb3
$ps –ef|grep pmon
$crsctl stat res -t
NOTE: At this point database instances and its related services are removed from cluster. ASM and other clusterware compontents will be removed as part of GRID infrastructure removal.
5) Disable and Stop Listener on the Node to be removed.
oracle@racattack18u::/u01 $ ps -ef|grep tns (To get the running listener_alias name)oracle@racattack18u::/u01 $ lsnrctl status <Listener Alias>oracle@racattack18u::/u01 $ srvctl config listener -a
-Use the below commands to disable the listener(s) which are running from ORACLE_HOMEoracle@racattack18u::/u01 $ srvctl disable listener -l <Listener Name> -n racattack18uoracle@racattack18u::/u01 $ srvctl stop listener -l <Listener Name> -n racattack18u
6) Update Inventory on the Node to be deleted
-Verify the inventory in racattack18u before updating it.
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
-Run the following command on racattack18u to update the inventory
$ /u01/products/rdbms_11203/oui/bin/runInstaller -updateNodeList ORACLE_HOME=/u01/products/rdbms_11203 "CLUSTER_NODES={racattack18u}" -local
-Verify the inventory in racattack18u it should not have information about racattack15u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
7.) Remove the RAC Database Software
The next step is to remove the RAC Database Software from the node to be deleted (racattack18u).
-Run below command as oracle user from racattack18u
$ cd $ORACLE_HOME/deinstall
$ ./deinstall -local
(-local option is to remoce ORACLE_HOME from local node)
8.) Update Inventory on the Remaining Nodes
-Verify the inventory in racattack15u before updating it.
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
-Run the following command on racattack15u to update the inventory
$ cd /u01/products/rdbms_11203/oui/bin
$ runInstaller -updateNodeList ORACLE_HOME=/u01/products/rdbms_11203 "CLUSTER_NODES={racattack15u}" -local
-Verify the inventory in racattack15u it should not have information about racattack18u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
9.) Remove Clusterware
The next step is to remove the clusterware from the node you want to delete.
Note: This must do as root.
-As root on racattack18u run the below command to deconfig clusterware
# cd /u01/products/grid_11203
# pwd
/u01/products/grid_11203
# cd crs/install
# pwd
/u01/products/grid_11203/crs/install
# hostname
racattack18u
# ./rootcrs.pl -deconfig -force
(At this point All the clusterware services including ASM are stopped)
10.) Delete node from Clusterware configuration
The next step is to remove the desired node from the clusterware configuration. This is done with the following commands.
# cd /u01/products/grid_11203/bin
# olsnodes –t –s (For verification)
-As root on racattack15u run the below command to delete racattack18u from cluster
# crsctl delete node -n racattack18u
(racattack18u will be removed from clusterware and registry)
-Run the below command from racattack15u to verify the existing nodes
# olsnodes –t –s (For verification)
(Should only show racattack15u)
11.) Update Inventory on node to be removed as "Grid"
The next step is to update the inventory on the desired node to be removed as the "Grid" software owner (Oracle for this environment).
Note: make sure that is done as the "Grid Owner" which is "Oracle" in this environment.
Verify the inventory in racattack18u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
-Run the following command on racattack18u to update the inventory
$ cd /u01/products/grid_11203/oui/bin
$ runInstaller -updateNodeList ORACLE_HOME=/u01/products/grid_11203 "CLUSTER_NODES={racattack18u}" -local
-Verify the inventory in racattack18u it should not have information about racattack15u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
12.) Remove the Clusterware Software from node to be removed.
The next step is to remove the clusterware software from the desired node (racattack18u).
Note: make sure that is done as the "Grid Owner". which is "Oracle" in this environment.
-Run below command as oracle user from racattack18u
$ cd /u01/products/grid_11203/deinstall
$ ./deinstall -local
(-local option is to remoce ORACLE_HOME from local node)
-In the end it asks us to run a script from a different session as root, run the script
13.) Update Inventory on remaining nodes for "Grid"
The next step is to update inventory on the remaining nodes (racattack15u) for "Grid".
Note: make sure that is done as the "Grid Owner"
Verify the inventory in racattack15u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
- Run the following command on racattack15u to update the inventory
$ cd /u01/products/grid_11203/oui/bin
$ ./runInstaller -updateNodeList ORACLE_HOME=/u01/products/grid_11203 "CLUSTER_NODES={racattack15u}"
-Verify the inventory in racattack15u it should not have information about racattack18u
$ cat /u01/app/oraInventory/ContentsXML/inventory.xml
14.) Confirm the node has been successfully removed.
The next step is to verify that the node has been successfully removed. To do this, execute the following command.
-Run the below command from racattack15u as oracle useroracle@racattack15u:racdb1:/u01 $ cluvfy stage -post nodedel -n racattack18uoracle@racattack15u:racdb1:/u01 $ crsctl stat res -t
NOTE: At this Stage racattack18u is clean without any Oracle software installations. racattack15u still remains as a single instance cluster database.
Saturday, July 4, 2015
How to set the diagwait parameter
Following is the action plan to set the diagwait parameter:
1) Stop database and other services across all the nodes
$srvctl stop database -d <db_name>
$srvctl stop asm -n node1
$srvctl stop asm -n node2
$srvctl stop nodeapps -n node1
$srvctl stop nodeapps -n node2
2) Stop the cluster on all the nodes
As root:
#crsctl stop crs
#<CRS_HOME>/bin/oprocd stop
3) Ensure that Clusterware stack is down on all nodes by executing
#ps -ef |egrep "crsd.bin|ocssd.bin|evmd.bin|oprocd"
4) From one of the node set the diagwait
As root:
#crsctl set css diagwait 13 -force
5) Check if diagwait is set successfully by executing, the following command.
#crsctl get css diagwait
6) Start cluster on all the nodes
As root:
#crsctl start crs
7) Check the status of cluster
crsctl check crs
8) Validate the status of all the services
crsstat.ksh
Steps to Configure VirtualBox Shared Folders in Ubuntu 10.10 Guest with Windows 7
1) Read this article and install VB guest additions.
2) Go to ‘Shared Folders’ option under virtual machine’s settings. Browse the Windows 7 folder or partition you want to share with Ubuntu 10.10.
Note: Any network between both Operating Systems not required.
Note down the Share Name. ‘Make Permanent’ option if you want all the time.
3) Guest OS will detect this share if guest additions installed and started properly. We have to mount this share inside Ubuntu guest. To do it, create a folder on any location.
I’m creating a folder inside media folder called ‘ windows7share’
sudo mkdir windows7share
4) Now we have to mount the VirtualBox shared folder to above folder with following command.
sudo mount -t vboxsf H_DRIVE /media/windows7share
5) That’s it. It should work without any issues. I could see the all folders and files from particular Windows 7 partition. Creating files and folders from Ubuntu guest worked well and I can see them from Windows 7 too, because ‘Read Only’ option was not selected.
6) Once you restart this virtual machine, the above mount will disappear. To make this mount and virtualbox shared folder access permanent,
a) Make sure to select ‘ Make Permanent’ option in step 2.
b) Add the command used in Step 4 to /etc/init.d/rc.local file, to run the command every time when systemstarts up.
Subscribe to:
Posts (Atom)