甜橘子，专注成都网站制作网站设计与营销型网站建设与优化
QQ :532337155
电话：028-86922220
第一章、基础环境部署
10多年的哈密网站建设经验，针对设计、前端、开发、售后、文案、推广等六对一服务，响应快，48小时及时工作处理。成都全网营销的优势是能够根据用户设备显示端的尺寸不同，自动调整哈密建站的显示方式，使网站能够适用不同显示终端，在浏览器中调整网站的宽度，无论在任何一种浏览器上浏览网站，都能展现优雅布局与设计，从而大程度地提升浏览体验。创新互联从事“哈密网站设计”,“哈密网站推广”以来，每个客户项目都认真落实执行。
操作系统：SuSE Linux Enterprise Server 11SP2
数据库：Oracle 11gR2(112040)
节点1
主机名：node1
ip地址：
192.168.1.30 node1
192.168.1.50 node1-vip
10.10.10.30 node1-priv
ORACLE_SID：ORACLE_1
节点2
主机名：node2
ip地址：
192.168.1.31 node2
192.168.1.51 node2-vip
10.10.10.31 node2-priv
ORACLE_SID：ORACLE_2
SCANIP: 192.168.1.41 my-cluster my-cluster-scan
共享存储：
设备名 | 大小 | 作用 |
/dev/sdb | 2g | OCR_VOTE |
/dev/sdc | 10g | 数据库归档日志 |
/dev/sdd | 20g | 数据库文件 |
1.2、创建账户
分别在两台服务器上创建如下账户。
groupadd-g 1000 oinstall
groupadd -g 1200 asmadmin
groupadd -g 1201 asmdba
groupadd -g 1202 asmoper
useradd -m -u 1100 -goinstall -G asmadmin,asmdba,asmoper -d /home/grid -s /bin/bash grid
groupadd -g 1300 dba
groupadd -g 1301oper
useradd -m -u 1101 -goinstall -G dba,oper,asmdba -d /home/oracle -s /bin/bash oracle
mkdir -p /u01/app/grid
mkdir -p /u01/app/11.2.0/grid
chown -R grid:oinstall/u01
mkdir -p /u01/app/oracle
chown -R oracle:oinstall/u01/app/oracle
chmod -R 775 /u01
passwd oracle
1.3、配置hosts文件
配置两台服务器hosts文件。
#vi /etc/hosts
192.168.1.30 node1
192.168.1.50 node1-vip
10.10.10.30 node1-priv
192.168.1.31 node2
192.168.1.51 node2-vip
10.10.10.31 node2-priv
192.168.1.41 my-cluster my-cluster-scan
1.4、安装所需rpm包
两节点均查看所需rpm包，安装所需rpm包。
#rpm -q binutils gcc gcc-32bit gcc-c++ glibc glibc-32bitglibc-devel glibc-devel-32bit ksh libaio libaio-32bit libaio-devellibaio-devel-32bit libstdc++33 libstdc++33-32bit libstdc++43 libstdc++43-32bitlibstdc++43-devel libstdc++43-devel-32bit libgcc43 libstdc++-devel make sysstatunixODBC unixODBC-devel unixODBC-32bit unixODBC-devel-32bit libcap1
binutils-2.23.1-0.17.18
gcc-4.3-62.198
gcc-32bit-4.3-62.198
gcc-c++-4.3-62.198
glibc-2.11.3-17.54.1
glibc-32bit-2.11.3-17.54.1
glibc-devel-2.11.3-17.54.1
glibc-devel-32bit-2.11.3-17.54.1
ksh-93u-0.18.1
libaio-0.3.109-0.1.46
libaio-32bit-0.3.109-0.1.46
libaio-devel-0.3.109-0.1.46
libaio-devel-32bit-0.3.109-0.1.46
libstdc++33-3.3.3-11.9
libstdc++33-32bit-3.3.3-11.9
libstdc++43-4.6.9-0.11.38
libstdc++43-32bit-4.6.9-0.11.38
libstdc++43-devel-4.3.4_20091019-0.37.30
libstdc++43-devel-32bit-4.3.4_20091019-0.37.30
libgcc43-4.6.9-0.11.38
libstdc++-devel-4.3-62.198
make-3.81-128.20
sysstat-8.1.5-7.45.24
unixODBC-2.2.12-198.17
unixODBC-devel-2.2.12-198.17
unixODBC-32bit-2.2.12-198.17
unixODBC-devel-32bit-2.2.12-198.17
libcap1-1.10-6.10
1.5、修改内核参数
设置两台服务器的内核参数
#vi /etc/sysctl.conf
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.conf.all.rp_filter = 1
fs.inotify.max_user_watches = 65536
net.ipv4.conf.default.promote_secondaries = 1
net.ipv4.conf.all.promote_secondaries = 1
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = 2097152
kernel.shmmax = 536870912
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
vm.hugetlb_shm_group= 1000 ###此处为oinstall group number
1.6、设置Shell对Oracle及grid用户的限制
vi /etc/security/limits.conf
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
grid soft nproc 2047
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
编辑/etc/pam.d/login，添加如下内容：
vi /etc/pam.d/login
session required pam_limits.so
1.7、禁用NTP服务
在两台服务器上均禁止NTP服务
service ntp stop
chkconfig ntp off
mv /etc/ntp.conf /etc/ntp.conf.bak
1.8、grid用户环境变量
节点1
ORACLE_SID=+ASM1; export ORACLE_SID
ORACLE_BASE=/u01/app/grid; export ORACLE_BASE
ORACLE_HOME=/u01/app/11.2.0/grid; export ORACLE_HOME
ORACLE_PATH=/u01/app/oracle/common/oracle/sql; exportORACLE_PATH
ORACLE_TERM=xterm; export ORACLE_TERM
TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN
ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK; export NLS_LANG
PATH=${PATH}:$HOME/bin:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH
LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH
export TEMP=/tmp
export TMPDIR=/tmp
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p16384
ulimit -n 65536
else
ulimit -u 16384 -n65536
fi
umask 022
fi
节点2
ORACLE_SID=+ASM2; export ORACLE_SID
ORACLE_BASE=/u01/app/grid; export ORACLE_BASE
ORACLE_HOME=/u01/app/11.2.0/grid; export ORACLE_HOME
ORACLE_PATH=/u01/app/oracle/common/oracle/sql; exportORACLE_PATH
ORACLE_TERM=xterm; export ORACLE_TERM
TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN
ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK; export NLS_LANG
PATH=${PATH}:$HOME/bin:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH
LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH
export TEMP=/tmp
export TMPDIR=/tmp
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p 16384
ulimit -n65536
else
ulimit -u 16384 -n65536
fi
umask 022
fi
1.9、oracle用户环境变量
节点1
ORACLE_SID=ORACLE_1; export ORACLE_SID
ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE
ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1; exportORACLE_HOME
ORACLE_PATH=/u01/app/common/oracle/sql; export ORACLE_PATH
ORACLE_TERM=xterm; export ORACLE_TERM
TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN
ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK; export NLS_LANG
PATH=${PATH}:$HOME/bin:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH
LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH
export TEMP=/tmp
export TMPDIR=/tmp
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p16384
ulimit -n65536
else
ulimit -u 16384 -n65536
fi
umask 022
fi
节点2
ORACLE_SID=ORACLE_2; export ORACLE_SID
ORACLE_BASE=/u01/app/oracle; export ORACLE_BASE
ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1; exportORACLE_HOME
ORACLE_PATH=/u01/app/common/oracle/sql; export ORACLE_PATH
ORACLE_TERM=xterm; export ORACLE_TERM
TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN
ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11
NLS_LANG=AMERICAN_AMERICA.ZHS16GBK; export NLS_LANG
PATH=${PATH}:$HOME/bin:$ORACLE_HOME/bin:$ORACLE_HOME/OPatch
PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin
PATH=${PATH}:/u01/app/common/oracle/bin
export PATH
LD_LIBRARY_PATH=$ORACLE_HOME/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib
LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib
export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib
CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib
export CLASSPATH
export TEMP=/tmp
export TMPDIR=/tmp
if [ $USER = "oracle" ] || [ $USER = "grid"]; then
if [ $SHELL ="/bin/ksh" ]; then
ulimit -p16384
ulimit -n65536
else
ulimit -u 16384 -n65536
fi
umask 022
fi
1.10、配置asm磁盘
node1:/etc/udev/rules.d# fdisk -l
Disk/dev/sda: 42.9 GB, 42949672960 bytes
255 heads, 63 sectors/track, 5221 cylinders, total 83886080 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x000b4c72
DeviceBoot Start End Blocks Id System
/dev/sda1 * 2048 401407 199680 83 Linux
/dev/sda2 401408 8787967 4193280 82 Linux swap / Solaris
/dev/sda3 8787968 83859455 37535744 83 Linux
Disk/dev/sdb: 2097 MB, 2097152000 bytes
65 heads, 62 sectors/track, 1016 cylinders, total 4096000 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk/dev/sdb doesn't contain a valid partition table
Disk/dev/sdc: 10.5 GB, 10485760000 bytes
64 heads, 32 sectors/track, 10000 cylinders, total 20480000 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk/dev/sdc doesn't contain a valid partition table
Disk/dev/sdd: 21.0 GB, 20971520000 bytes
64 heads, 32 sectors/track, 20000 cylinders, total 40960000 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000
Disk/dev/sdd doesn't contain a valid partition table
node1:/etc/udev/rules.d #
添加编辑/etc/scsi_id.config 文件，并同步至节点2上。
options=--whitelisted --replace-whitespace
添加编辑/etc/udev/rules.d/99-oracle-asmdevices.rules
node1:~ # for i in b c d
> do
> echo "KERNEL==\"sd*\", SUBSYSTEM==\"block\",PROGRAM==\"/lib/udev/scsi_id --whitelisted --replace-whitespace--device=/dev/\
$name\",RESULT==\"`/lib/udev/scsi_id --whitelisted --replace-whitespace--device=/dev/sd$i`\", NAME=\"asm-disk$i\", OWNER=\"grid
\",GROUP=\"asmadmin\", MODE=\"0660\""
> done
KERNEL=="sd*", SUBSYSTEM=="block",PROGRAM=="/lib/udev/scsi_id --whitelisted --replace-whitespace--device=/dev/$name",
RESULT=="2e8ced45878991f7f",NAME="asm-diskb", OWNER="grid", GROUP="asmadmin",MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block",PROGRAM=="/lib/udev/scsi_id --whitelisted --replace-whitespace--device=/dev/$name",
RESULT=="2ff5dd7058b0ffb40",NAME="asm-diskc", OWNER="grid", GROUP="asmadmin",MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block",PROGRAM=="/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",
RESULT=="26c6d14a2e3aa5dd4",NAME="asm-diskd", OWNER="grid", GROUP="asmadmin",MODE="0660"
修改设备别名为所需设备名，同步至节点2.
node1:~ # cd /etc/udev/rules.d/
node1:/etc/udev/rules.d # vi 99-oracle-asmdevices.rules
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM=="/lib/udev/scsi_id--whitelisted --replace-whitespace --device=/dev/$name",
RESULT=="2e8ced45878991f7f",NAME="asmocr", OWNER="grid", GROUP="asmadmin",MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block",PROGRAM=="/lib/udev/scsi_id --whitelisted --replace-whitespace--device=/dev/$name",
RESULT=="2ff5dd7058b0ffb40",NAME="asmfra", OWNER="grid", GROUP="asmadmin",MODE="0660"
KERNEL=="sd*", SUBSYSTEM=="block",PROGRAM=="/lib/udev/scsi_id --whitelisted --replace-whitespace--device=/dev/$name",
RESULT=="26c6d14a2e3aa5dd4",NAME="asmdata", OWNER="grid", GROUP="asmadmin",MODE="0660"
"99-oracle-asmdevices.rules" 3L, 625Cwritten
node1:/etc/udev/rules.d #
SUSE和RedHat系列不同,需要使用udevadm test测试才可以看到新产生的设备名称，节点2也需进行测试。
node1:/etc/udev/rules.d# udevadm test /sys/block/sdb
node1:/etc/udev/rules.d # udevadm test /sys/block/sdc
node1:/etc/udev/rules.d # udevadm test /sys/block/sdd
重新启动udev(确保udev为运行状态)，节点2页重新启动。
node1:/etc/udev/rules.d # /etc/init.d/boot.udev restart
Restarting udevd: done
node1:/etc/udev/rules.d #
查看生成的设备名称
node1:/dev # ll asm*
brw-rw---- 1 grid asmadmin 8, 48 May 22 23:11 asmdata
brw-rw---- 1 grid asmadmin 8, 32 May 22 23:11 asmfra
brw-rw---- 1 grid asmadmin 8, 16 May 22 23:11 asmocr
node1:/dev #