apiVersion: route.openshift.io/v1
kind: Route
metadata:
  name: example-route
spec:
  to:
    name: example-service
  port:
    targetPort: 8080
  tls:
    termination: edge
#定义了一个名为 example-route 的 Route，它将 example-service 服务暴露给外部用户。tls 字段指定了 SSL 终止的方式，这里我们使用了 Edge 终止方式。这意味着 OpenShift Router 会在自己内部处理 SSL，而不是将 SSL 流量转发给后端服务。

apiVersion: v1
kind: ImageStream
metadata:
  name: my-image-stream
spec:
  dockerImageRepository: registry.example.com/my-image
  #创建一个名为“my-image-stream”的ImageStream对象，它将使用“registry.example.com/my-image”作为Docker镜像仓库。您可以使用类似的代码来创建和管理其他类型的Registry对象，例如ImageStreamTag和ImageStreamImport。

在OpenShift中，与 Registry 相关的对象主要包括ImageStream、ImageStreamTag和ImageStreamImport。以下是每个对象的简要说明：

- ImageStream：ImageStream是一个抽象对象，用于表示一个或多个容器镜像的流。它可以用来管理和共享容器镜像，并提供版本控制和回滚功能。您可以使用ImageStream来创建和管理容器镜像的流，以及将容器镜像推送到Registry中。
- ImageStreamTag：ImageStreamTag是ImageStream的子对象，用于表示一个特定版本的容器镜像。它可以用来管理和共享容器镜像的版本，并提供版本控制和回滚功能。您可以使用ImageStreamTag来创建和管理容器镜像的版本，并将其推送到Registry中。
- ImageStreamImport：ImageStreamImport是一个对象，用于将外部Registry中的容器镜像导入到OpenShift中的Registry中。它可以用来管理和共享外部Registry中的容器镜像，并提供版本控制和回滚功能。您可以使用ImageStreamImport来将外部Registry中的容器镜像导入到OpenShift中的Registry中，并将其用作ImageStream或ImageStreamTag的基础镜像。

一个常见的疑问是：是不是OpenShift用到的镜像都要存放到内置的仓库？ 答案是否定的。内部的镜像仓库存放的只是S2I产生的镜像。其他镜像可以存放在集群外部的镜像仓库，如企业的镜像仓库或社区的镜像仓库。只要保证OpenShift的节点可以访问到这些镜像所在的镜像仓库即可。

vi /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
IPADDR=172.16.183.90
NETMASK=255.255.255.0
GATEWAY=172.16.183.2
DNS1=172.16.183.2
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
DEVICE=ens33
ONBOOT=yes

lsblk -f #查看刚添加的盘是否识别
fdisk /dev/sdb #n，后面保持默认，分一个sdb1出来，w保存
mkfs -t xfs /dev/sdb1 #格式化
#查看新添加的disk大小
fdisk -l
blkid /dev/sdb1 #获取UUID
vim /etc/fstab
#添加永久挂载
UUID=xxx /mnt/data xfs defaults 0 1
mount -a #执行挂载

Host ocpmaster1
  HostName ocpmaster1
  User root
  Port 22

#修改VM ssh配置文件
vim /etc/ssh/sshd_config
#修改GSSAPIAuthentication no
#取消注释 PubkeyAuthentication yes
#宿主机上复制ssh公钥
cat ~/.ssh/id_rsa.pub
#VM上创建authorized_keys文件
mkdir -p ~/.ssh/
vi ~/.ssh/authorized_keys
#将host主机的公钥粘贴进去保存，即可用VSCode免密登录VM

yum update -y

yum install -y device-mapper-persistent-data lvm2 wget net-tools nfs-utils lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo ntp libaio-devel wget vim ncurses-devel autoconf automake zlib-devel  python-devel epel-release openssh-server socat  ipvsadm conntrack telnet ipvsadm

sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
reboot -f

hostnamectl set-hostname ocpmaster1 && bash 
hostnamectl set-hostname ocpnode1 && bash
hostnamectl set-hostname ocpnode2 && bash

vim /etc/hosts
172.16.183.90   ocpmaster
172.16.183.91   ocpnode1  
172.16.183.92   ocpnode2

ssh-keygen
ssh-copy-id ocpmaster1
ssh-copy-id ocpnode1
ssh-copy-id ocpnode2

#临时关闭
swapoff -a
#永久关闭：注释swap挂载，给swap这行开头加一下注释
vim /etc/fstab
#/dev/mapper/centos-swap swap      swap    defaults        0 0

modprobe br_netfilter
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
sysctl -p /etc/sysctl.d/k8s.conf #sysctl -p：从指定的文件加载系统参数，如不指定即从/etc/sysctl.conf中加载

systemctl stop firewalld && systemctl disable firewalld

yum install yum-utils -y
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

#安装ntpdate命令
yum install ntpdate -y
#跟网络时间做同步
ntpdate cn.pool.ntp.org
#把时间同步做成计划任务
crontab -e
* */1 * * * /usr/sbin/ntpdate   cn.pool.ntp.org
#重启crond服务
service crond restart

vim /etc/sysconfig/network-scripts/ifcfg-ens33
#最后追加如下内容：
NM_CONTROLLED=yes
#修改好之后重启网络
service network restart

#停掉NetworkManager
systemctl stop NetworkManager && systemctl disable NetworkManager

#把ansible-2.6.5-1.el7.ans.noarch.rpm上传到master、node1、node2上
yum install ansible-2.6.5-1.el7.ans.noarch.rpm -y

#把openshift-ansible-release-3.10.zip上传到master节点，手动解压：
unzip openshift-ansible-release-3.10.zip

#3台都需要配置
yum install -y docker-1.13.1

#修改docker配置文件
vim /etc/sysconfig/docker
#把之前的OPTIONS注释掉 
#OPTIONS='--selinux-enabled --log-driver=journald --signature-verification=false'
#新增如下内容：
OPTIONS='--selinux-enabled  --signature-verification=False'

#配置docker镜像加速器
systemctl start docker
vim /etc/docker/daemon.json 
{"registry-mirrors": ["http://6e9e5b27.m.daocloud.io","https://registry.docker-cn.com"],
"insecure-registries":["172.16.183.90:5000"]
}
systemctl restart docker

#仅在master上配置私有镜像仓库
docker load -i registry.tar.gz
#使用httpd作为身份提供商
yum install httpd -y
systemctl start httpd
systemctl enable httpd
mkdir -p /opt/registry-var/auth/
#设置registry的用户名密码
docker run --entrypoint htpasswd registry:2.5 -Bbn hangxu hangxu  >> /opt/registry-var/auth/htpasswd

#设置配置文件
mkdir -p /opt/registry-var/config
vim /opt/registry-var/config/config.yml
version: "0.1"
log:
  fields:
    service: registry
storage:
  delete:
    enabled: true
  cache:
    blobdescriptor: inmemory
  filesystem:
    rootdirectory: /var/lib/registry
http:
  addr: :5000
  headers:
    X-Content-Type-Options: [nosniff]
health:
  storagedriver:
    enabled: true
    interval: 10s
threshold: 3

#启动服务
docker run -d -p 5000:5000 --restart=always  --name=registry -v /opt/registry-var/config/:/etc/docker/registry/ -v /opt/registry-var/auth/:/auth/ -e "REGISTRY_AUTH=htpasswd"  -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd -v /opt/registry-var/:/var/lib/registry/ registry:2.5

#3台上都测试一下
docker login 172.16.183.90:5000
username：hangxu
pwd：hangxu

#设置docker开机自启动
systemctl enable docker
systemctl is-active docker
#所有节点更改/etc/sysconfig/docker-storage-setup如下：
vim /etc/sysconfig/docker-storage-setup
DEVS=/dev/sdb 
VG=docker-vg
#所有Node节点执行docker-storage-setup
docker-storage-setup 
##!有这个提示：INFO: Storage is already configured with overlay2 driver. Can't configure it with devicemapper driver. To override, remove /etc/sysconfig/docker-storage and retry.

#master解压
docker load -i openshift_master_3_10.tar.gz
#node解压
docker load -i openshift_slave_3_10.tar.gz

#在master上
vim /etc/ansible/hosts 
[OSEv3:children]
masters
nodes
etcd
[OSEv3:vars]
openshift_deployment_type=origin
ansible_ssh_user=root
ansible_become=yes
openshift_repos_enable_testing=true
openshift_enable_service_catalog=false
template_service_broker_install=false
debug_level=4
openshift_clock_enabled=true
openshift_version=3.10.0 
openshift_image_tag=v3.10
openshift_disable_check=disk_availability,docker_storage,memory_availability,docker_image_availability,os_sdn_network_plugin_name=redhat/openshift-ovs-multitenant i
openshift_master_identity_providers=[{'name': 'htpasswd_auth','login': 'true', 'challenge': 'true','kind': 'HTPasswdPasswordIdentityProvider'}]
[masters] 
ocpmaster1
[nodes]
ocpmaster1 openshift_node_group_name='node-config-master'
ocpnode1 openshift_node_group_name='node-config-master' 
ocpnode2 openshift_node_group_name='node-config-master'
[etcd]
ocpmaster1

#安装前预配置检查，在master上执行
ansible-playbook -i /etc/ansible/hosts openshift-ansible-release-3.10/playbooks/prerequisites.yml
#显示ok 没有报错即可安装
ansible-playbook -i /etc/ansible/hosts openshift-ansible-release-3.10/playbooks/deploy_cluster.yml

echo nameserver 8.8.8.8 >>/etc/resolv.conf

crontab -e
* */1 * * * /usr/sbin/ntpdate   cn.pool.ntp.org
* * * * * /usr/bin/echo  nameserver 8.8.8.8  >>/etc/resolv.conf

TASK [openshift_manage_node : Set node schedulability]
#到这个task之后执行下面部分
oc label node ocpnode1 node-role.kubernetes.io/infra=true
oc label node ocpnode2 node-role.kubernetes.io/infra=true
#显示complete表示安装成功

#首次新建用户密码
htpasswd -cb /etc/origin/master/htpasswd admin admin
#添加另一个用户密码
htpasswd -b /etc/origin/master/htpasswd dev dev
#以集群管理员登录
oc login -u system:admin
#给用户admin分配一个cluster-admin角色
oc adm policy add-cluster-role-to-user cluster-admin admin
#浏览器登录openshift，配置自己电脑hosts文件，添加：
172.16.183.90 ocpmaster1

#Master上执行如下，获取admin密码
oc login -n openshift
#admin/admin
oc whoami -t
#sDNr30KlWs-_NyxKhqhOAyOheLz56FGBA_c_mKT2d38
#每个节点执行如下登陆openshift私有镜像仓库    
docker login docker-registry.default.svc:5000
#用户：admin
#密码：sDNr30KlWs-_NyxKhqhOAyOheLz56FGBA_c_mKT2d38

oc label nodes ocpnode1 node-role.kubernetes.io/compute=true
oc label nodes ocpnode2 node-role.kubernetes.io/compute=true
oc label nodes ocpnode1 node-role.kubernetes.io/infra=true
oc label nodes ocpnode2 node-role.kubernetes.io/infra=true

oc get pods -n myproject

oc get service -n myproject

oc new-project dev --display-name="Tasks - Dev" 
oc new-project stage --display-name="Tasks - Stage" 
oc new-project cicd --display-name="CI/CD"

oc policy add-role-to-group edit system:serviceaccounts:cicd -n dev 
oc policy add-role-to-group edit system:serviceaccounts:cicd -n stage
oc policy add-role-to-group view system:serviceaccounts:cicd -n dev 
oc policy add-role-to-group view system:serviceaccounts:cicd -n stage

#3个节点上都执行：
oc login -n openshift
#master上执行，获取密码
oc whoami -t
#每个节点执行如下，登录私有仓库
docker login docker-registry.default.svc:5000
#用户：admin
#密码：前一步生成的密码

docker pull registry.access.redhat.com/openshift3/jenkins-agent-maven-35-rhel7

open /etc/docker/certs.d/registry.access.redhat.com/redhat-ca.crt: no such file or directory

#在每个节点操作
yum install *rhsm* -y
wget http://mirror.centos.org/centos/7/os/x86_64/Packages/python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm
rpm2cpio python-rhsm-certificates-1.19.10-1.el7_4.x86_64.rpm | cpio -iv --to-stdout ./etc/rhsm/ca/redhat-uep.pem | tee /etc/rhsm/ca/redhat-uep.pem

oc label node node1 node-role.kubernetes.io/compute=true --overwrite
oc label node master node-role.kubernetes.io/compute=true --overwrite
oc label node node2 node-role.kubernetes.io/compute=true --overwrite

#在master上操作
oc process -f cicd-template.yaml -n cicd | oc create -f - -n cicd
oc get pods -n cicd

#jenkins账号密码：使用openshift账号密码登陆
#sonarqube账号密码：admin/admin
#nexus账号密码：admin/admin123
#gogs账号密码：需要注册，gogs/gogs

oc log <pod name>
#nginx: [emerg] mkdir() "/var/cache/nginx/client_temp" failed (13: Permission denied)

oc debug <pod name> -c <container name>
ls -la /var/cache
#drwxr-xr-x. 2 root root  6 May 29 16:45 nginx
#发现nginx目录只能由root来写，但是openshift中容器是没有root权限的
whoami
#user ID 1000110000

docker pull nginxinc/nginx-unprivileged:latest

#oc命令是带有权限管控的，所以在使用 oc 命令进行实际的操作前，需要先通过 oc 1ogin 命令登录
oc login -u developer
Logged into "https://127.0.0.1:8443" as "developer" using existing credentials.

You have access to the following projects and can switch between them with 'oc project <projectname>':

    hello-world
  * myproject

Using project "myproject".

#oc rsh 命令在 OpenShift 3 中用于启动一个新的 shell 会话并连接到指定的容器中。这使得你可以在容器内部执行命令，进行调试或者查看容器的运行状态。这个命令类似于 Docker 的 exec 命令。
#使用 oc rsh 的基本语法如下：
oc rsh <pod-name>

oc project <project name>
#查看当前project
oc project

#在命令行可以通过 oc new-app 命令方便地部署 DockerHub 等 Docker 镜像仓库的镜像。只提供了镜像，其他的 API 资源都是自动生成的。
docker pull openshift/hello-openshift
oc new-app openshift/hello-openshift #--name podName 指定 应用名称

whereis oc # kubectl: /usr/bin/oc
echo "alias k=/usr/bin/oc">>/etc/profile
source /etc/profile
#2. 安装并设置alias的自动补全：
yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
oc completion bash | sudo tee /etc/bash_completion.d/oc > /dev/null
##生成 bash 自动补全脚本，写入到 /etc/bash_completion.d/kubectl 文件中
source <(oc completion bash | sed 's/oc/k/g')
#3. 解决每次启动k都会失效，要重新刷新环境变量（source /etc/profile）的问题：在~/.bashrc文件中添加以下代码：source /etc/profile
echo "source /etc/profile" >> ~/.bashrc
#4. 给alias k也配置补全
vim ~/.bashrc
##加入以下
alias k='oc'
complete -o default -F __start_oc k
source <(oc completion bash)
##使命令生效
source ~/.bashrc

tee cluster-role-namespace-admin.yaml <<'EOF'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: namespace-admin
rules:
- apiGroups: ["*"]
  resources: ["*"]
  verbs: ["*"]
EOF

# Apply the yaml file
oc apply -f cluster-role-namespace-admin.yaml

# Take project opt-airflow as a demo
tee binding-namespace-admin.yaml <<'EOF'
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: admin-opt-airflow
  namespace: opt-airflow # Specify the name of project/namespace
subjects:
- kind: Group
  name: team-test
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: namespace-admin
  apiGroup: rbac.authorization.k8s.io
EOF

# Apply the yaml file
oc apply -f binding-namespace-admin.yaml

kubectl get pod -owide | grep <pod name>

oc debug node/<nodename>
# It is NOT required to run chroot /host and toolbox in currently supported OpenShift versions. The oc debug node/<nodename> has tcpdump already.

NAME=<pod-name>
NAMESPACE=<pod-namespace>
pod_id=$(chroot /host crictl pods --namespace ${NAMESPACE} --name ${NAME} -q)
ns_path="/host$(chroot /host bash -c "crictl inspectp $pod_id | jq '.info.runtimeSpec.linux.namespaces[]|select(.type==\"network\").path' -r")"
nsenter_parameters="--net=${ns_path}"

nsenter $nsenter_parameters -- chroot /host ip a

nsenter $nsenter_parameters -- tcpdump -nn -i ${INTERFACE} -w /host/var/tmp/${HOSTNAME}_$(date +\%d_%m_%Y-%H_%M_%S-%Z).pcap ${TCPDUMP_EXTRA_PARAMS}

ls /host/var/tmp/*.pcap

kubectl get po -A | grep debug
oc project <debugPodNamespace>

$ oc get pods
NAME                                            READY   STATUS      RESTARTS   AGE
ip-10-z-yyy-xxxus-west-2computeinternal-debug   1/1     Running     0          97s

$ oc cp ip-10-z-yyy-xxxus-west-2computeinternal-debug:/host/var/tmp/<filename>.pcap <filename>.pcap