JImmy的文档： 搭建集群版openshift origin-1.5.0

系统安装：

       系统配置，理论上一般配置都可以，推荐

                 内存 4G以上

                 CPU 4个以上

                 硬盘 100G以上。

       系统镜像： CentOS-7-x86_64-DVD-1611.iso 

       安装分组为： 带GUI的服务器



需要准备四台服务器，分别安装上面的系统和分组，并配置ip 和主机名，主机名如下：

                   master.origin.com.cn   

                   etcd.origin.com.cn

                   node01.origin.com.cn

                   node02.origin.com.cn

在每台主机的/et/hosts 目录下添加主机名对应IP的路由表，/etc/hosts内容如下（在所有主机上运行）：

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.1.110.76 master.origin.com.cn
10.1.110.77 etcd.origin.com.cn
10.1.110.78 node01.origin.com.cn
10.1.110.79 node02.origin.com.cn

分别添加各个主机对master主机的信任（master主机上运行）：

[root@master ~]#  ssh-keygen
[root@master ~]#  ssh-copy-id -i ~/.ssh/id_rsa.pub master.origin.com.cn
[root@master ~]#  ssh-copy-id -i ~/.ssh/id_rsa.pub etcd.origin.com.cn
[root@master ~]#  ssh-copy-id -i ~/.ssh/id_rsa.pub node01.origin.com.cn
[root@master ~]#  ssh-copy-id -i ~/.ssh/id_rsa.pub node02.origin.com.cn

在每个主机上都配置yum源（在所有主机上运行）：

[root@master ~] cd /etc/yum.repos.d/
[root@master ~] wget http://210.76.97.70/repo/openshift/openshift.repo

master节点上安装：

[root@master ~]yum install openshift-master-allinone.x86_64

在etd节点上安装：

[root@etcd ~]# yum install openshift-etcd-allinone.x86_64

在node节点上安装：

[root@node01 ~]# yum install openshift-node-allinone.x86_64

设置Docker的registry的访问网络范围（在所有主机上运行）：

[root@master ~]sed -i '/OPTIONS=.*/c\OPTIONS="--log-driver=journald --insecure-registry 172.30.0.0/16"' /etc/sysconfig/docker  
[root@master ~]systemctl restart docker

在master节点进行批量安装部署：

  配置ansible文件：

[root@master ~]cat /etc/ansible/hosts
# Create an OSEv3 group that contains the masters, nodes, and etcd groups
[OSEv3:children]
   masters
   nodes
   etcd

# Set variables common for all OSEv3 hosts
[OSEv3:vars]
  ansible_ssh_user=root
  deployment_type=origin

[masters]
  master.origin.com.cn

# host group for etcd
[etcd]
  etcd.origin.com.cn

# host group for nodes, includes region info
[nodes]
  master.origin.com.cn openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
  node01.origin.com.cn openshift_node_labels="{'region': 'primary', 'zone': 'east'}"
  node02.origin.com.cn openshift_node_labels="{'region': 'primary', 'zone': 'west'}"

#测试ping：

[root@master ~]#  ansible all -m ping
[root@master ~]# ansible all -m ping
etcd.origin.com.cn | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
node01.origin.com.cn | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
node02.origin.com.cn | SUCCESS => {
    "changed": false,
    "ping": "pong"
}
master.origin.com.cn | SUCCESS => {
    "changed": false,
    "ping": "pong"
}

#使用ansible 部署环境

[root@master ~]# ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/byo/config.yml

PLAY RECAP *********************************************************************
etcd.origin.com.cn         : ok=114  changed=40   unreachable=0    failed=0
localhost                  : ok=10   changed=0    unreachable=0    failed=0
master.origin.com.cn       : ok=531  changed=147  unreachable=0    failed=0
node01.origin.com.cn       : ok=222  changed=63   unreachable=0    failed=0
node02.origin.com.cn       : ok=222  changed=63   unreachable=0    failed=0

[root@master ~]# oc get nodes
NAME                   STATUS                     AGE
master.origin.com.cn   Ready,SchedulingDisabled   1h
node01.origin.com.cn   Ready                      1h
node02.origin.com.cn   Ready                      1h

#将master纳入node节点（可选）

[root@master ~]# oadm manage-node master.origin.com.cn --schedulable=true
NAME                   STATUS    AGE
master.origin.com.cn   Ready     1h

[root@master ~]# oc get nodes
NAME                   STATUS    AGE
master.origin.com.cn   Ready     1h
node01.origin.com.cn   Ready     1h
node02.origin.com.cn   Ready     1h

#脚本默认安装了域名服务器dnsmasq，需将自己定义的域名加入配置文件（在master和node节点上都要运行）。

[root@master ~]# cat /etc/dnsmasq.d/origin-dns.conf
strict-order
no-resolv
domain-needed
server=/cluster.local/172.30.0.1
server=/30.172.in-addr.arpa/172.30.0.1
address=/.origin.com.cn/10.1.110.76

[root@master ~]systemctl stop dnsmasq
[root@master ~]systemctl restart NetworkManager
[root@master ~]systemctl stop iptables

#在每个主机/etc/resolv.conf中，添加 "nameserver 10.1.110.76"
[root@node01 ~]# cat /etc/resolv.conf
# Generated by NetworkManager
nameserver 10.1.110.76
nameserver 202.106.0.20

测试功能
[root@server113 ~]# nslookup origin.com.cn

Server:        10.1.110.76
Address:    10.1.110.76#53

Name:    origin.com.cn
Address: 10.1.110.76

[root@server113 ~]# nslookup node01.origin.com.cn
Server:        10.1.110.76
Address:    10.1.110.76#53

Name:    node01.origin.com.cn
Address: 10.1.110.78

登录：

[root@master ~]# oc login -u system:admin -n default
Logged into "https://master.origin.com.cn:8443" as "system:admin" using existing credentials.

You have access to the following projects and can switch between them with 'oc project <projectname>':

* default
kube-system
logging
management-infra
openshift
openshift-infra

Using project "default".

这些namespaces在界面登陆默认是看不到的，可以执行如下给admin赋权。
[root@master ~]# oadm policy add-cluster-role-to-user cluster-admin admin
 

新建registry：

#默认脚本会安装registry，我们先将其清空
[root@master ~]# oc get bc
[root@master ~]# oc get dc
[root@master ~]# oc get svc
NAME         CLUSTER-IP   EXTERNAL-IP   PORT(S)                 AGE
kubernetes   172.30.0.1   <none>        443/TCP,53/UDP,53/TCP   9d
[root@master ~]# oc get rc
[root@master ~]# oc get pods
[root@master ~]# oc get serviceaccounts
NAME       SECRETS   AGE
builder    2         9d
default    3         9d
deployer   2         9d
[root@master ~]#

#新建registry
[root@master ~]# mkdir -p /opt/openshift-registry
[root@master ~]# chown 1001:root /opt/openshift-registry
[root@master ~]# oc create serviceaccount registry -n default
[root@master ~]# oadm policy add-scc-to-user privileged system:serviceaccount:default:registry
[root@master ~]# oadm registry --service-account=registry --mount-host=/opt/openshift-registry
error: serviceaccounts "registry" already exists
error: rolebinding "registry-registry-role" already exists
deploymentconfig "docker-registry" created
service "docker-registry" created

#查看状态
[root@master ~]# oc get svc
NAME              CLUSTER-IP       EXTERNAL-IP   PORT(S)                 AGE
docker-registry   172.30.118.103   <none>        5000/TCP                1m
kubernetes        172.30.0.1       <none>        443/TCP,53/UDP,53/TCP   9d
[root@master ~]# oc get dc
NAME              REVISION   REPLICAS   TRIGGERED BY
docker-registry   1          1          config
[root@master ~]# oc get pods
NAME                      READY     STATUS    RESTARTS   AGE
docker-registry-1-dkxxq   1/1       Running   0          1m

#查看日志：
[root@master ~]# oc logs dc/docker-registry
time="2016-07-15T08:22:58.216681281Z" level=info msg="version=v2.1.0+unknown"
time="2016-07-15T08:22:58.261312307Z" level=info msg="redis not configured" go.version=go1.6 instance.id=45bc8e12-07d4-4503-b4a5-8f1ffabb2295
time="2016-07-15T08:22:58.261368151Z" level=info msg="Starting upload purge in 11m0s" go.version=go1.6 instance.id=45bc8e12-07d4-4503-b4a5-8f1ffabb2295
time="2016-07-15T08:22:58.26143377Z" level=info msg="using inmemory blob descriptor cache" go.version=go1.6 instance.id=45bc8e12-07d4-4503-b4a5-8f1ffabb2295
time="2016-07-15T08:22:58.261508162Z" level=info msg="Using Origin Auth handler"
time="2016-07-15T08:22:58.262252207Z" level=debug msg="configured \"openshift\" access controller" go.version=go1.6 instance.id=45bc8e12-07d4-4503-b4a5-8f1ffabb2295
time="2016-07-15T08:22:58.262553713Z" level=info msg="listening on :5000" go.version=go1.6 instance.id=45bc8e12-07d4-4503-b4a5-8f1ffabb2295
10.1.1.1 - - [15/Jul/2016:08:23:03 +0000] "GET /healthz HTTP/1.1" 200 0 "" "Go 1.1 package http"
10.1.1.1 - - [15/Jul/2016:08:23:05 +0000] "GET /healthz HTTP/1.1" 200 0 "" "Go 1.1 package http"
10.1.1.1 - - [15/Jul/2016:08:23:15 +0000] "GET /healthz HTTP/1.1" 200 0 "" "Go 1.1 package http"
10.1.1.1 - - [15/Jul/2016:08:23:15 +0000] "GET /healthz HTTP/1.1" 200 0 "" "Go 1.1 package http"
....

#将user账户赋权（可访问registry，可上传下载镜像）
[root@master ~]# oadm policy add-role-to-user system:registry user
[root@master ~]# oadm policy add-role-to-user admin user -n openshift
[root@master ~]# oadm policy add-role-to-user system:image-builder user

#使用user用户登陆
[root@master ~]# oc login
Authentication required for https://master.openshift.cmiot.com:8443 (openshift)
Username: user
Password:
Login successful.

Using project "openshift".

#查看该用户的token
[root@master ~]# oc whoami -t
CHRljvdJVjc9NbSYa7swlV21U-WAeW89JxD29o0Lwfg

#使用token登陆仓库
[root@master ~]# docker login -u user -p CHRljvdJVjc9NbSYa7swlV21U-WAeW89JxD29o0Lwfg -e 123@qq.com 172.30.118.103:5000
WARNING: login credentials saved in /root/.docker/config.json
Login Succeeded

#上传镜像至私有仓库的openshift
[root@master ~]# docker pull alpine
[root@master ~]# docker tag alpine 172.30.118.103:5000/openshift/alpine
[root@master ~]# docker push 172.30.118.103:5000/openshift/alpine
The push refers to a repository [172.30.118.103:5000/openshift/alpine]
4fe15f8d0ae6: Pushed
latest: digest: sha256:ca7591b00591f1ec2513b919db9ed2f7ece11a0b166ee507a62be9bfd583764b size: 1497

#再看看可否上传至default（提示500错误，说明没有权限）
[root@master ~]# docker tag alpine 172.30.118.103:5000/default/alpine
[root@master ~]# docker push 172.30.118.103:5000/default/alpine
The push refers to a repository [172.30.118.103:5000/default/alpine]
4fe15f8d0ae6: Pushed
Received unexpected HTTP status: 500 Internal Server Error

#node01下载镜像
[root@master ~]# oc get is -n openshift
NAME                 DOCKER REPO                                        TAGS                         UPDATED
alpine               172.30.118.103:5000/openshift/alpine               latest                       2 hours ago
....

[root@node01 ~]# docker login -u user -p CHRljvdJVjc9NbSYa7swlV21U-WAeW89JxD29o0Lwfg -e 123@qq.com 172.30.118.103:5000
WARNING: login credentials saved in /root/.docker/config.json
Login Succeeded
[root@node01 ~]# docker pull 172.30.118.103:5000/openshift/alpine
Using default tag: latest
Trying to pull repository 172.30.118.103:5000/openshift/alpine ...
latest: Pulling from 172.30.118.103:5000/openshift/alpine
e110a4a17941: Pull complete
Digest: sha256:ca7591b00591f1ec2513b919db9ed2f7ece11a0b166ee507a62be9bfd583764b
Status: Downloaded newer image for 172.30.118.103:5000/openshift/alpine:latest
[root@node01 ~]#

固定registry的ip

创建registry的svc时ip地址是随机，如果再次创建ip会变，可以使用yaml文件部署将ip固定。

#获取原svc的cluster IP
[root@master ~]# oc get svc
NAME              CLUSTER-IP       EXTERNAL-IP   PORT(S)                 AGE
docker-registry   172.30.118.103   <none>        5000/TCP                17h
kubernetes        172.30.0.1       <none>        443/TCP,53/UDP,53/TCP   10d
[root@master ~]#  oc get svc/docker-registry -o yaml | grep clusterIP:
clusterIP: 172.30.118.103

#将现有的registry保存为yaml文件
[root@master ~]# oadm registry --service-account=registry --mount-host=/opt/openshift-registry --selector="region=infra" -o yaml > /root/template/registry.yaml

#查看yaml文件
[root@master ~]# more /root/template/registry.yaml
...
- apiVersion: v1
kind: Service
metadata:
creationTimestamp: null
labels:
docker-registry: default
name: docker-registry
spec:
clusterIP: 172.30.118.103
portalIP: 172.30.118.103
ports:
- name: 5000-tcp
port: 5000
targetPort: 5000
selector:
docker-registry: default
....

#验证yaml文件（先删除原svc，再根据yaml创建）
#删除原svc
[root@master ~]# oc delete svc/docker-registry dc/docker-registry
service "docker-registry" deleted
deploymentconfig "docker-registry" deleted

#使用yaml创建svc
[root@master ~]# oc create -f /root/template/registry.yaml
deploymentconfig "docker-registry" created
service "docker-registry" created
Error from server: serviceaccounts "registry" already exists
Error from server: rolebinding "registry-registry-role" already exists
[root@master ~]# oc get svc
NAME              CLUSTER-IP       EXTERNAL-IP   PORT(S)                 AGE
docker-registry   172.30.118.103   <none>        5000/TCP                9m
kubernetes        172.30.0.1       <none>        443/TCP,53/UDP,53/TCP   10d
[root@master ~]# oc get pods
NAME                      READY     STATUS    RESTARTS   AGE
docker-registry-1-qb6uj   1/1       Running   0          9m

#验证push和pull功能
[root@master ~]# oc login
Authentication required for https://master.openshift.cmiot.com:8443 (openshift)
Username: user
Password:
Login successful.

Using project "openshift".
[root@master ~]# oc whoami -t
T5a4KqjSaAHYj-4Ue4mAr7FnvsrPueCgfDST6OjRm9s
[root@master ~]# docker login -u user -p T5a4KqjSaAHYj-4Ue4mAr7FnvsrPueCgfDST6OjRm9s -e 123@qq.com 172.30.118.103:5000
WARNING: login credentials saved in /root/.docker/config.json
Login Succeeded
#push功能
[root@master ~]# docker pull ubuntu
[root@master ~]# docker tag ubuntu 172.30.118.103:5000/openshift/ubuntu
[root@master ~]# docker push 172.30.118.103:5000/openshift/ubuntu
The push refers to a repository [172.30.118.103:5000/openshift/ubuntu]
eb33f051084b: Pushed
3e907de58acd: Pushed
b04095771c04: Pushed
3d865816fc94: Pushed
latest: digest: sha256:09544931f08488c001d4008a10718d422d0969745f02ab4d15b0c35d8b3b4273 size: 4783

#node01下载
[root@node01 ~]# docker login -u user -p T5a4KqjSaAHYj-4Ue4mAr7FnvsrPueCgfDST6OjRm9s -e 123@qq.com 172.30.118.103:5000
WARNING: login credentials saved in /root/.docker/config.json
Login Succeeded
[root@node01 ~]# docker pull 172.30.118.103:5000/openshift/ubuntu
Using default tag: latest
Trying to pull repository 172.30.118.103:5000/openshift/ubuntu ...
latest: Pulling from 172.30.118.103:5000/openshift/ubuntu
90d6565b970a: Pull complete
40553bdb8474: Pull complete
c3129e7479ab: Pull complete
091663bd70db: Pull complete
Digest: sha256:09544931f08488c001d4008a10718d422d0969745f02ab4d15b0c35d8b3b4273
Status: Downloaded newer image for 172.30.118.103:5000/openshift/ubuntu:latest

新建route：

#新建serviceaccout，并加入SCC
[root@master ~]# oc create serviceaccount router -n default
[root@master ~]# oadm policy add-scc-to-user privileged system:serviceaccount:default:router

#新建router
[root@master ~]# oadm router router --replicas=1 --service-account=router

#查看状态
[root@master ~]# oc get svc
NAME              CLUSTER-IP       EXTERNAL-IP   PORT(S)                   AGE
docker-registry   172.30.118.103   <none>        5000/TCP                  1d
kubernetes        172.30.0.1       <none>        443/TCP,53/UDP,53/TCP     12d
router            172.30.2.17      <none>        80/TCP,443/TCP,1936/TCP   39s
[root@master ~]# oc get pods -o wide
NAME                      READY     STATUS    RESTARTS   AGE       NODE
docker-registry-1-qb6uj   1/1       Running   0          1d        master.openshift.cmiot.com
router-1-ewcju            1/1       Running   0          15m       node01.openshift.cmiot.com

#查看node01，发现多了几个haproxy进程
[root@node01 ~]# netstat -tnlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 127.0.0.1:10443         0.0.0.0:*               LISTEN      34921/haproxy
tcp        0      0 127.0.0.1:10444         0.0.0.0:*               LISTEN      34921/haproxy
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      34921/haproxy
tcp        0      0 0.0.0.0:1936            0.0.0.0:*               LISTEN      34921/haproxy
tcp        0      0 0.0.0.0:53              0.0.0.0:*               LISTEN      49621/dnsmasq
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1199/sshd
tcp        0      0 0.0.0.0:443             0.0.0.0:*               LISTEN      34921/haproxy
tcp6       0      0 :::10250                :::*                    LISTEN      2377/openshift
tcp6       0      0 :::53                   :::*                    LISTEN      49621/dnsmasq
tcp6       0      0 :::22                   :::*                    LISTEN      1199/sshd

#查看日志
[root@master ~]# oc logs dc/router
I0717 15:17:29.831718       1 router.go:161] Router is including routes in all namespaces
I0717 15:17:30.436138       1 router.go:310] Router reloaded:
- Checking HAProxy /healthz on port 1936 ...
- HAProxy port 1936 health check ok : 0 retry attempt(s).
I0717 15:17:34.876811       1 router.go:310] Router reloaded:
- Checking HAProxy /healthz on port 1936 ...
- HAProxy port 1936 health check ok : 0 retry attempt(s).
I0717 15:17:39.875319       1 router.go:310] Router reloaded:
- Checking HAProxy /healthz on port 1936 ...
- HAProxy port 1936 health check ok : 0 retry attempt(s).
I0717 15:17:48.782875       1 router.go:310] Router reloaded:
- Checking HAProxy /healthz on port 1936 ...
- HAProxy port 1936 health check ok : 0 retry attempt(s).

#动态扩容router的副本数
[root@master ~]#  oc scale dc/router --replicas=2
deploymentconfig "router" scaled
[root@master ~]# oc get pods -o wide
NAME                      READY     STATUS    RESTARTS   AGE       NODE
docker-registry-1-qb6uj   1/1       Running   0          1d        master.openshift.cmiot.com
router-1-ewcju            1/1       Running   0          55m       node01.openshift.cmiot.com
router-1-px9ss            1/1       Running   0          21m       node02.openshift.cmiot.com

#将router部署在master主机（可选）
[root@master ~]# oadm router router00 --replicas=1 --service-account=router --selector="region=infra"