Deploying UPI OKD 4.5 Cluster

yum install -y vim bind bind-utils

vim /etc/named.conf

vim /var/named/dynamic/forward.db

vim /var/named/dynamic/reverse.db

systemctl enable named
systemctl restart named
systemctl status named

firewall-cmd   --permanent --add-service=dns --zone=public  --permanent
firewall-cmd --reload

#Change DNS Server
nmtui

#Restart interface
ifdown ens3;ifup en3

nslookup api.openshift.riupie.io

yum install dhcp-server -y

vim /etc/dhcp/dhcpd.conf

systemctl enable dhcpd
systemctl restart dhcpd
systemctl status dhcpd

sudo firewall-cmd --add-service=dhcp --permanent
sudo firewall-cmd --reload

yum install tftp-server ipxe-bootimgs -y

ln -s /usr/share/ipxe/undionly.kpxe /var/lib/tftpboot

curl -LO https://github.com/poseidon/matchbox/releases/download/v0.8.3/matchbox-v0.8.3-linux-amd64.tar.gz
tar xvzf matchbox-v0.8.3-linux-amd64.tar.gz 

cd matchbox-v0.8.3-linux-amd64
cp matchbox /usr/local/bin/

useradd -U matchbox

mkdir -p /var/lib/matchbox/{assets,groups,ignition,profiles}

chown -R matchbox:matchbox /var/lib/matchbox

systemctl enable tftp
systemctl restart tftp

firewall-cmd --permanent --add-service=tftp
firewall-cmd --reload

cp contrib/systemd/matchbox-local.service /etc/systemd/system/matchbox.service

systemctl daemon-reload
systemctl enable matchbox
systemctl restart matchbox

cd /var/lib/matchbox/assets

wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200907.3.0/x86_64/fedora-coreos-32.20200907.3.0-live-kernel-x86_64 -O fcos-32-kernel
wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200907.3.0/x86_64/fedora-coreos-32.20200907.3.0-live-kernel-x86_64.sig -O fcos-32-kernel.sig

wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200907.3.0/x86_64/fedora-coreos-32.20200907.3.0-live-initramfs.x86_64.img -O fcos-32-initramfs.img
wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200907.3.0/x86_64/fedora-coreos-32.20200907.3.0-live-initramfs.x86_64.img.sig -O fcos-32-initramfs.img.sig 

wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200907.3.0/x86_64/fedora-coreos-32.20200907.3.0-metal.x86_64.raw.xz -O fcos-32-metal.raw.xz
wget https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200907.3.0/x86_64/fedora-coreos-32.20200907.3.0-metal.x86_64.raw.xz.sig -O fcos-32-metal.raw.xz.sig

vim /var/lib/matchbox/profiles/bootstrap.json

{
"id": "bootstrap",

"name": "OKD 4.5 - Bootstrap",

"ignition_id": "bootstrap.ign",

"boot": {

"kernel": "/assets/fcos-32-kernel",

"initrd": ["/assets/fcos-32-initramfs.img"],

"args": [

"ip=dhcp",

"rd.neednet=1",

"console=tty0",

"console=ttyS0",

"coreos.inst=yes",

"coreos.inst.install_dev=vda",

"coreos.inst.image_url=http://helper.openshift.riupie.io:8080/assets/fcos-32-metal.raw.xz",

"coreos.inst.ignition_url=http://helper.openshift.riupie.io:8080/ignition?mac=${mac:hexhyp}"
]

}
}

vim /var/lib/matchbox/profiles/master.json

{
"id": "master",

"name": "OKD 4.5 - Master",

"ignition_id": "master.ign",

"boot": {

"kernel": "/assets/fcos-32-kernel",

"initrd": ["/assets/fcos-32-initramfs.img"],

"args": [

"ip=dhcp",

"rd.neednet=1",

"console=tty0",

"console=ttyS0",

"coreos.inst=yes",

"coreos.inst.install_dev=vda",

"coreos.inst.image_url=http://helper.openshift.riupie.io:8080/assets/fcos-32-metal.raw.xz",

"coreos.inst.ignition_url=http://helper.openshift.riupie.io:8080/ignition?mac=${mac:hexhyp}"

]

}

}

vim /var/lib/matchbox/profiles/worker.json

{
"id": "worker",

"name": "OKD 4.5 - Worker",

"ignition_id": "worker.ign",

"boot": {

"kernel": "/assets/fcos-32-kernel",

"initrd": [

"/assets/fcos-32-initramfs.img"

],

"args": [

"ip=dhcp",

"rd.neednet=1",

"console=tty0",

"console=ttyS0",

"coreos.inst=yes",

"coreos.inst.install_dev=vda",

"coreos.inst.image_url=http://helper.openshift.riupie.io:8080/assets/fcos-32-metal.raw.xz",

"coreos.inst.ignition_url=http://helper.openshift.riupie.io:8080/ignition?mac=${mac:hexhyp}"

]

}

}

vim /var/lib/matchbox/groups/bootstrap.json

{
"id": "bootstrap01",

"name": "OCP 4.5 - Bootstrap server",

"profile": "bootstrap",

"selector": {

"mac": "52:54:00:7a:55:a1"

}

}

vim /var/lib/matchbox/groups/master01.json 

{
"id": "master01",

"name": "OCP 4.5 - Master 1",

"profile": "master",

"selector": {

"mac": "52:54:00:58:b0:a9"

}

}

vim /var/lib/matchbox/groups/master02.json 

{
"id": "master02",

"name": "OCP 4.5 - Master 2",

"profile": "master",

"selector": {

"mac": "52:54:00:79:70:08"

}

}

vim /var/lib/matchbox/groups/master03.json 

{
"id": "master03",

"name": "OCP 4.5 - Master 3",

"profile": "master",

"selector": {

"mac": "52:54:00:de:92:a2"

}

}

vim /var/lib/matchbox/groups/worker01.json 

{
"id": "worker01",

"name": "OCP 4.5 - Worker 1",

"profile": "worker",

"selector": {

"mac": "52:54:00:6f:4b:1f"

}

}

vim /var/lib/matchbox/groups/worker02.json 

{
"id": "worker02",

"name": "OCP 4.5 - Worker 2",

"profile": "worker",

"selector": {

"mac": "52:54:00:c7:f4:21"

}

}

chown -R matchbox:matchbox /var/lib/matchbox

firewall-cmd --permanent --add-port 8080/tcp
firewall-cmd --reload

systemctl restart matchbox

yum install -y chrony

vim /etc/chrony.conf

systemctl enable chronyd
systemctl restart chronyd
systemctl status chronyd

chronyc sources

timedatectl set-timezone Asia/Jakarta

firewall-cmd --permanent --add-service=ntp
firewall-cmd --reload

#Change DNS Server
nmtui

#Restart interface
ifdown ens3;ifup en3

# Install chrony
yum install chrony -y

# Set NTP Server
vim /etc/chrony.conf

#Add this line
server 10.10.50.9 iburst

# Restart chrony
systemctl restart chronyd

[root@okd-lb1 ~]# timedatectl
               Local time: Mon 2020-10-26 19:28:16 WIB
           Universal time: Mon 2020-10-26 12:28:16 UTC
                 RTC time: Mon 2020-10-26 12:28:16
                Time zone: Asia/Jakarta (WIB, +0700)
System clock synchronized: yes
              NTP service: active
          RTC in local TZ: no

yum update -y
yum install -y keepalived

mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bk
vim /etc/keepalived/keepalived.conf

systemctl enable keepalived
systemctl start keepalived
systemctl status keepalived

yum install -y haproxy rsyslog

vim /etc/haproxy/haproxy.cfg

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

    # utilize system-wide crypto-policies
    ssl-default-bind-ciphers PROFILE=SYSTEM
    ssl-default-server-ciphers PROFILE=SYSTEM

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend openshift-api-server
    bind api.openshift.riupie.io:6443
    default_backend openshift-api-server
    mode tcp
    option tcplog
frontend machine-config-server
    bind api-int.openshift.riupie.io:22623
    default_backend machine-config-server
    mode tcp
    option tcplog
frontend ingress-http
    bind *:80
    default_backend ingress-http
    mode tcp
    option tcplog
frontend ingress-https
    bind *:443
    default_backend ingress-https
    mode tcp
    option tcplog

#---------------------------------------------------------------------
# static backend for serving up API, MSC, HTTP and HTTPS
#---------------------------------------------------------------------
backend openshift-api-server
    balance source
    mode tcp
    server bootstrap.openshift.riupie.io 10.10.51.10:6443 check
    server master01.openshift.riupie.io 10.10.51.11:6443 check
    server master02.openshift.riupie.io 10.10.51.12:6443 check
    server master03.openshift.riupie.io 10.10.51.13:6443 check
backend machine-config-server
    balance source
    mode tcp
    server bootstrap.openshift.riupie.io 10.10.51.10:22623 check
    server master01.openshift.riupie.io 10.10.51.11:22623 check
    server master02.openshift.riupie.io 10.10.51.12:22623 check
    server master03.openshift.riupie.io 10.10.51.13:22623 check
backend ingress-http
    balance source
    mode tcp
    server worker01.openshift.riupie.io 10.10.51.21:80 check
    server worker02.openshift.riupie.io 10.10.51.22:80 check
backend ingress-https
    balance source
    mode tcp
    server worker01.openshift.riupie.io 10.10.51.21:443 check
    server worker02.openshift.riupie.io 10.10.51.22:443 check

vim /etc/rsyslog.conf

vim /etc/rsyslog.d/haproxy.conf

setsebool -P haproxy_connect_any=1

sysctl -w net.ipv4.ip_nonlocal_bind=1

systemctl enable haproxy
systemctl restart haproxy
systemctl status haproxy

systemctl enable rsyslog
systemctl restart rsyslog
systemctl status rsyslog

firewall-cmd --permanent --add-service http
firewall-cmd --permanent --add-service https
firewall-cmd --permanent --add-port 6443/tcp
firewall-cmd --permanent --add-port 22623/tcp
firewall-cmd --reload

yum install -y nfs-utils

systemctl start nfs-server.service
systemctl enable nfs-server.service
systemctl status nfs-server.service

mkdir -p  /mnt/nfs_shares/okd
chmod -R 777 /mnt/nfs_shares/okd/

vim /etc/exports

exportfs -arv

exportfs  -s

firewall-cmd --permanent --add-service=nfs
firewall-cmd --permanent --add-service=nfs3
firewall-cmd --permanent --add-service=rpc-bind
firewall-cmd --permanent --add-service=mountd
firewall-cmd --reload

mkfs.xfs /dev/vdb

df -hT

#Generate SSH keypair
ssh-keygen

#Optional: Save ssh private key using ssh-agent if you use passphrase.
eval "$(ssh-agent -s)"
ssh-add /root/.ssh/id_rsa

#Download openshift client
wget https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz

#Extract package
tar xvzf /root/oc.tar.gz

#Move oc binary to /usr/bin/
mv /root/oc /usr/bin/

#Download openshift-installer
wget https://github.com/openshift/okd/releases/download/4.5.0-0.okd-2020-09-18-202631/openshift-install-linux-4.5.0-0.okd-2020-09-18-202631.tar.gz

#Extract openshift-installer package
tar xvzf /root/openshift-install-linux-4.5.0-0.okd-2020-09-18-202631.tar.gz

#Create a directory to save openshift configuration
mkdir /root/okd-config
mv /root/openshift-install /root/okd-config/
cd /root/okd-config

vim /root/okd-config/install-config.yaml

apiVersion: v1
baseDomain: riupie.io
compute:
- hyperthreading: Enabled
  name: worker
  replicas: 0
controlPlane:
  hyperthreading: Enabled
  name: master 
  replicas: 3
metadata:
  name: openshift 
networking:
  clusterNetwork:
  - cidr: 10.128.0.0/14 
    hostPrefix: 23 
  networkType: OpenShiftSDN
  serviceNetwork: 
  - 172.30.0.0/16
platform:
  none: {} 
pullSecret: '{"auths":{"fake":{"auth": "bar"}}}' 
sshKey: 'your_bastion_server_ssh_pub_key'

cp /root/okd-config/install-config.yaml /root/okd-config/install-config.yaml.bk

cd /root/okd-config/
./openshift-install create manifests --dir=/root/okd-config/

vim /root/okd-config/manifests/cluster-scheduler-02-config.yml

apiVersion: config.openshift.io/v1
kind: Scheduler
metadata:
  creationTimestamp: null
  name: cluster
spec:
  mastersSchedulable: false
  policy:
    name: ""
status: {}

./openshift-install create ignition-configs --dir=/root/okd-config/

/root/okd-config/
├── auth
│   ├── kubeadmin-password
│   └── kubeconfig
├── bootstrap.ign
├── master.ign
├── openshift-install
└── worker.ign

cp /root/okd-config/*.ign /var/lib/matchbox/ignition/

chown -R matchbox:matchbox /var/lib/matchbox

systemctl restart matchbox

[root@okd02-bastion okd-config]# ./openshift-install --dir=/root/okd-config/ wait-for bootstrap-complete --log-level=info
INFO Waiting up to 20m0s for the Kubernetes API at https://api.openshift.riupie.io:6443... 
INFO API v1.18.3 up                               
INFO Waiting up to 40m0s for bootstrapping to complete... 
INFO It is now safe to remove the bootstrap resources 
INFO Time elapsed: 0s

vim /etc/haproxy/haproxy.cfg

./openshift-install --dir=/root/okd-config/ wait-for install-complete --log-level=debug


DEBUG Still waiting for the cluster to initialize: Working towards 4.5.0-0.okd-2020-09-18-202631: 87% complete, waiting on authentication, monitoring
DEBUG Cluster is initialized
INFO Waiting up to 10m0s for the openshift-console route to be created...
DEBUG Route found in openshift-console namespace: console
DEBUG Route found in openshift-console namespace: downloads
DEBUG OpenShift console route is created
INFO Install complete!
INFO To access the cluster as the system:admin user when using 'oc', run 'export KUBECONFIG=/root/okd-config/auth/kubeconfig'
INFO Access the OpenShift web-console here: https://console-openshift-console.apps.openshift.riupie.io
INFO Login to the console with user: "kubeadmin", and password: "KjV4r-eg3Uq-qEtU5-AxPS6"
DEBUG Time elapsed per stage:
DEBUG Cluster Operators: 9m8s
INFO Time elapsed: 9m8s

export KUBECONFIG=/root/okd-config/auth/kubeconfig

#Check user
oc whoami

oc get nodes

oc get csr
oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}' | xargs oc adm certificate approve

oc get nodes

oc get pods --all-namespaces

vim /etc/hosts

10.10.51.100 console-openshift-console.apps.openshift.riupie.io oauth-openshift.apps.openshift.riupie.io alertmanager-main-openshift-monitoring.apps.openshift.riupie.io

yum install git -y
git clone https://github.com/kubernetes-incubator/external-storage.git kubernetes-incubator

oc create namespace openshift-nfs-storage

oc label namespace openshift-nfs-storage "openshift.io/cluster-monitoring=true"

# Switch project
oc project openshift-nfs-storage

# Change namespace on deployment and rbac YAML file
cd /root/kubernetes-incubator/nfs-client/

NAMESPACE=`oc project -q`

sed -i'' "s/namespace:.*/namespace: $NAMESPACE/g" ./deploy/rbac.yaml 
sed -i'' "s/namespace:.*/namespace: $NAMESPACE/g" ./deploy/deployment.yaml

oc create -f deploy/rbac.yaml
oc adm policy add-scc-to-user hostmount-anyuid system:serviceaccount:$NAMESPACE:nfs-client-provisioner

vim /root/kubernetes-incubator/nfs-client/deploy/deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: openshift-nfs-storage
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: storage.io/nfs
            - name: NFS_SERVER
              value: 10.10.51.9           # Change this
            - name: NFS_PATH
              value: /mnt/nfs_shares/okd  # Change this
      volumes:
        - name: nfs-client-root
          nfs:
            server: 10.10.51.9            # Change this
            path: /mnt/nfs_shares/okd     # Change this

vim /root/kubernetes-incubator/nfs-client/deploy/class.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-nfs-storage
provisioner: storage.io/nfs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
  archiveOnDelete: "false"

oc create -f deploy/class.yaml 
oc create -f deploy/deployment.yaml

oc get pods -n openshift-nfs-storage

oc patch storageclass managed-nfs-storage -p '{"metadata": {"annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}}'

oc patch configs.imageregistry.operator.openshift.io cluster --type merge --patch '{"spec":{"managementState":"Managed"}}'

oc get pod -n openshift-image-registry

oc edit configs.imageregistry.operator.openshift.io

...
spec:
.
.
.
  storage:
    pvc:
      claim:
...

oc get clusteroperator image-registry

oc get pvc --all-namespaces

NAMESPACE                  NAME                     STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS          AGE
openshift-image-registry   image-registry-storage   Bound    pvc-bf06735c-df5e-4b3c-9aba-f0e3bf598d80   100Gi      RWX            managed-nfs-storage   96s

watch -n5 oc get clusteroperators

NAME                                       VERSION                         AVAILABLE   PROGRESSING   DEGRADED   SINCE
authentication                             4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
cloud-credential                           4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
cluster-autoscaler                         4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
config-operator                            4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
console                                    4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
csi-snapshot-controller                    4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
dns                                        4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
etcd                                       4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
image-registry                             4.5.0-0.okd-2020-09-18-202631   True        False         False	3m5s
ingress                                    4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
insights                                   4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
kube-apiserver                             4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
kube-controller-manager                    4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
kube-scheduler                             4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
kube-storage-version-migrator              4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
machine-api                                4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
machine-approver                           4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
machine-config                             4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
marketplace                                4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
monitoring                                 4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
network                                    4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
node-tuning                                4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
openshift-apiserver                        4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
openshift-controller-manager               4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
openshift-samples                          4.5.0-0.okd-2020-09-18-202631   True        True          True	11d
operator-lifecycle-manager                 4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
operator-lifecycle-manager-catalog         4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
operator-lifecycle-manager-packageserver   4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
service-ca                                 4.5.0-0.okd-2020-09-18-202631   True        False         False	11d
storage                                    4.5.0-0.okd-2020-09-18-202631   True        False         False	11d