How to create an Embassy-Hosted Kubernetes Cluster with OpenStack Magnum

openstack coe cluster template list -f value | grep ehk

792819b6-5eb8-4740-b7d3-95295399bdb8 ehk-basic-template-v1.0

openstack coe cluster create test-k8s-cluster \
                     --cluster-template ehk-basic-template-v1.0 \
                     --node-count 3 \
                     --keypair [your-keypair] \
                     --master-count 1

openstack coe nodegroup list test-k8s-cluster
openstack coe nodegroup show test-k8s-cluster default-worker --fit-width
openstack coe cluster resize test-k8s-cluster --nodegroup default-worker 3
openstack coe nodegroup update test-k8s-cluster default-worker replace min_node_count=3

resource "openstack_containerinfra_cluster_v1" "test-cluster" {
  provider            = openstack
  name                = "test-cluster"
  cluster_template_id = "04d9e633-ef89-435b-ae9d-8cffa2aae330"
  keypair             = "my-keypair"
  node_count          = 1
  flavor              = "1c1m20d"
  master_count        = 1
  master_flavor       = "1c2m20d"
  fixed_network       = "my-network"
  fixed_subnet        = "my-subnet"
}

# Export resulting kubeconfig file

resource "local_file" "kubconfig_file" {
  depends_on          = [openstack_containerinfra_cluster_v1.my-k8s]
  filename            = "./kubeconfig"
  file_permission     = 600
  content             = openstack_containerinfra_cluster_v1.my-k8s.kubeconfig.raw_config
}

grep 'client-certificate-data' ./config | awk '{print $2}' | base64 -d | openssl x509 -text | grep O=
        Subject: CN=admin, O=system:masters

Kind: Service
apiVersion: v1
metadata:
  name: test
  namespace: default
spec:
  type: LoadBalancer
  loadBalancerSourceRanges:
    - 193.62.192.0/21
    - 10.0.0.0/24

openstack coe cluster list -f json
[
  {
    "status": "CREATE_COMPLETE",
    "uuid": "fc8e21eb-34bf-47a4-8170-faed729f2da9",
    "health_status": "HEALTHY",
    "master_count": 1,
    "keypair": "magnum-test",
    "node_count": 1,
    "name": "vac-test"
  }
]

openstack loadbalancer listener list -f json | jq '.[]|select(.protocol_port==6443)'
{
  "protocol_port": 6443,
  "protocol": "TCP",
  "name": "vac-test-dgb3dcidunvz-api_lb-3afx5fzx6lkd-listener-h3xt3lecsond",
  "admin_state_up": true,
  "default_pool_id": "feb8f1a0-3513-4c44-8eaa-2fbf91ce7d91",
  "project_id": "deb06b667e454f8a9e005bf334b6e65e",
  "id": "0a08414c-4f68-4d59-9386-9beb116017f4"
}

openstack subnet show 505151c1-09b9-4c9c-a349-fd348420188a -f json | jq .cidr
"10.0.0.0/24"

openstack loadbalancer listener set  --allowed-cidr 193.62.192.0/21 --allowed-cidr 10.0.0.0/24 vac-test-dgb3dcidunvz-api_lb-3afx5fzx6lkd-listener-h3xt3lecsond

openstack loadbalancer listener show vac-test-dgb3dcidunvz-api_lb-3afx5fzx6lkd-listener-h3xt3lecsond -f json | jq .allowed_cidrs
"10.0.0.0/24\n193.62.192.0/21"

openstack loadbalancer show vac-test-dgb3dcidunvz-api_lb-3afx5fzx6lkd-loadbalancer-wct5fhalbuid -c vip_port_id -f value
bcae96b3-7d22-49fa-bfc7-f53acec0efd2

openstack floating ip list -f json | jq '.[]|select(.Port=="bcae96b3-7d22-49fa-bfc7-f53acec0efd2")|."Floating IP Address"'
"45.88.80.193"

nc -z -v 45.88.80.193 6443
Connection to 45.88.80.193 6443 port [tcp/*] succeeded!

kubectl get sc
NAME                       PROVISIONER                RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
default-cinder (default)   cinder.csi.openstack.org   Delete          Immediate           true                   67m

git clone https://github.com/kubernetes-sigs/nfs-ganesha-server-and-external-provisioner
docker login https://dockerhub.ebi.ac.uk -u [gitlab user] -p [gitlab personal access token]
cd nfs-ganesha-server-and-external-provisioner
make build
make container
docker tag   nfs-provisioner:latest dockerhub.ebi.ac.uk/[your gitlab project]/[your gitlab repo]/nfs-provisioner
docker push dockerhub.ebi.ac.uk/[your gitlab project]/[your gitlab repo]/nfs-provisioner
kubectl create secret docker-registry regcred --docker-server=dockerhub.ebi.ac.uk --docker-username=[your gitlab user] --docker-password=[gitlab personal access token]

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: nfsclaim
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
  storageClassName: default-cinder

    image: dockerhub.ebi.ac.uk/[your gitlab project]/[your gitlab repo]/nfs-provisioner
imagePullSecrets:
  - name: regcred
volumes:
  - name: export-volume
    persistentVolumeClaim:
      claimName: nfsclaim

kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
  name: example-nfs
provisioner: example.com/nfs
reclaimPolicy: Retain
mountOptions:
  - vers=4.1

kubectl create -f deploy/kubernetes/deployment.yaml
kubectl create -f deploy/kubernetes/rbac.yaml
kubectl create -f deploy/kubernetes/class.yaml
kuˆbectl create -f deploy/kubernetes/claim.yaml

kubectl get sc
NAME                       PROVISIONER                RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
default-cinder (default)   cinder.csi.openstack.org   Delete          Immediate           true                   6h49m
example-nfs                example.com/nfs            Delete          Immediate           false                  73m

kubectl get pvc
NAME                   STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS     AGE
NFSClaim               Bound    pvc-69e1c626-6628-4ec6-b738-415658cbde2f   2Gi        RWO            default-cinder   5h28m
nfs                    Bound    pvc-33ca8e4c-9c97-4a03-b91f-f953a31721a8   1Mi        RWX            example-nfs      65m

kubectl edit pvc my-claim

kubectl get pv

openstack volume show 8218e1d7-cea9-4b78-8e51-281ff95904b7 -c size

kubectl get pvc

kubectl get svc ingress-nginx-controller  -n ingress-nginx
NAME                       TYPE           CLUSTER-IP      EXTERNAL-IP    PORT(S)                      AGE
ingress-nginx-controller   LoadBalancer   10.254.247.42   45.88.80.167   80:31666/TCP,443:31920/TCP   19m

---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: myservice-ingress
  namespace: myservice
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/whitelist-source-range: 193.62.0.0/16
spec:
  tls:
    - hosts:
      secretName: myservice.ebi.ac.uk
  rules:
    - host: myservice.ebi.ac.uk
      http:
        paths:
          - backend:
              serviceName: myservice-service
              servicePort: 8000
            path: /

kubectl create deployment nginx --image=nginx
kubectl create service nodeport nginx --tcp=80:80 --node-port 31389
kubectl get svc
NAME              TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                                                                                                     AGE
nginx             NodePort    10.254.137.50   <none>        80:31389/TCP

openstack subnet list -c ID -c Name -f value

6882959c-0db5-4f50-ad2d-d05fbfeaf654 test-cems-bcntwzytb3so-network-ul7yucjqajfr-private_subnet-6tpafqbzhn44                                                                                             5s

openstack loadbalancer create --vip-subnet-id 6882959c-0db5-4f50-ad2d-d05fbfeaf654 --name test-cems-lb

openstack loadbalancer listener create --protocol-port 31389 --name test-cems-listener --protocol TCP test-cems-lb

openstack loadbalancer pool create --listener test-cems-listener  --protocol TCP --lb-algorithm SOURCE_IP --name cems-test-pool

openstack server list
+--------------------------------------+---------------------------------+--------+---------------------+----------------+-----------+
| ID                                   | Name                            | Status | Networks            | Image          | Flavor    |
+--------------------------------------+---------------------------------+--------+---------------------+----------------+-----------+
| 35ffe989-e3a9-4e3e-89e0-5715bf868938 | test-cems-bcntwzytb3so-node-0   | ACTIVE | test-cems=10.0.0.77 | FedoraCoreOS32 | m1.medium |
| 041a5c6a-0290-4deb-95be-070ef617a232 | test-cems-bcntwzytb3so-master-0 | ACTIVE | test-cems=10.0.0.32 | FedoraCoreOS32 | m1.medium |
+--------------------------------------+---------------------------------+--------+---------------------+----------------+-----------+

openstack loadbalancer member create --address 10.0.0.77  --protocol-port 31389 cems-test-pool

openstack floating ip create public

openstack loadbalancer show -c vip_port_id test-cems-lb
+-------------+--------------------------------------+
| Field       | Value                                |
+-------------+--------------------------------------+
| vip_port_id | 703014b9-3872-44ea-8f3a-477cd2569250 |
+-------------+--------------------------------------+

openstack floating ip set --port 703014b9-3872-44ea-8f3a-477cd2569250 f59f98b2-9b80-458d-ad12-048af49294e4

curl [floating IP address]:31389
\...e
<h1>Welcome to nginx!</h1>
...../

kubectl create secret generic my-ssh-key --from-file=id_rsa=/home/cems/rsynctestk8s/test.pem

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rwmdata
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 1Gi
  storageClassName: example-nfs

apiVersion: batch/v1
kind: Job
metadata:
  name: rsync-$ITEM
spec:
  template:
    spec:
      containers:
      - name: rsync-$ITEM
        image: quay.io/embassycloud/alpinersync:latest
        imagePullPolicy: Always
        command: ["rsync", "-a","-v","-e", "ssh -i /mykey/id_rsa -o StrictHostKeyChecking=no", "centos@[ip address]:/home/centos/test-transfer/$ITEM", "/data"]
        volumeMounts:
        - mountPath: /data
          name: data
        - mountPath: "/mykey"
          readOnly: true
          name: ssh-key-volume
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: rwmdata
      - name: ssh-key-volume
        secret:
          secretName: my-ssh-key
          defaultMode: 256
      restartPolicy: Never
  backoffLimit: 1

mkdir ./jobs
for i in path1 path2 path3
do
  cat job-tmpl.yaml | sed "s/\$ITEM/$i/" > ./jobs/job-$i.yaml
done

kubectl create -f ./jobs

kubectl delete job rsync

openssl req -new -newkey rsa:4096 -nodes -keyout myuser.key -out myuser.csr \
    -subj "/CN=myuser/O=myproject"

cat <<EOF | kubectl apply -f -
  apiVersion: certificates.k8s.io/v1
  kind: CertificateSigningRequest
  metadata:
      name: myuser-myproject
  spec:
      groups:
      - system:authenticated
      request: $(cat myuser.csr | base64 | tr -d '\n')
      signerName: kubernetes.io/kube-apiserver-client
      usages:
      - client auth
EOF

  certificatesigningrequest.certificates.k8s.io/myuser-myproject created

  # Check status
  kubectl get csr
  NAME               AGE   REQUESTOR    CONDITION
  myuser-myproject   3s    kube-admin   Pending

kubectl certificate approve myuser-myproject
certificatesigningrequest.certificates.k8s.io/myuser-myproject approved

# Check current Status
kubectl get csr
NAME               AGE   REQUESTOR    CONDITION
myuser-myproject   27s   kube-admin   Approved,Issued

kubectl get csr myuser-myproject -o jsonpath='{.status.certificate}' \
    | base64 --decode > myuser-myproject.crt

kubectl config view -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' --raw  \
    | base64 --decode - > k8s-ca.crt

$ kubectl create namespace myproject
namespace/myproject created

# Create new kubeconfig file with given credentials and a new context
CLUSTER=$(kubectl config view -o jsonpath='{.clusters[0].name}')
SERVER=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}')

kubectl config set-cluster $CLUSTER --server=$SERVER --certificate-authority=k8s-ca.crt --kubeconfig=myuser-config --embed-certs
Cluster "local" set.

kubectl config set-credentials myuser  --cluster=$CLUSTER --client-certificate=myuser-myproject.crt  --client-key=myuser.key         --kubeconfig=myuser-config
User "myuser" set.

kubectl config set-context myuser-myproject-context --cluster=$CLUSTER --namespace=myproject --user=myuser  --kubeconfig=myuser-config
Context "myuser-myproject-context" created.

kubectl config use-context myuser-myproject-context --kubeconfig=myuser-config
Switched to context "myuser-myproject-context".

# Check context
kubectl config get-contexts --kubeconfig=myuser-config
CURRENT   NAME                       CLUSTER   AUTHINFO   NAMESPACE
*         myuser-myproject-context   local     myuser     myproject

# This user doesn't have access to resource nodes
kubectl get nodes --kubeconfig=myuser-config
Error from server (Forbidden): nodes is forbidden: User "myuser" cannot list resource "nodes" in API group "" at the cluster scope

# Give the user cluster admin access
kubectl create clusterrolebinding myuser-cluster-admin --clusterrole=cluster-admin --user=myuser

# List Cluster Role Binding
kubectl get clusterrolebinding myuser-cluster-admin
NAME                   AGE
myuser-cluster-admin   12s

# Test you can get node status using this kubeconfig file
kubectl get nodes --kubeconfig=myuser-config
NAME               STATUS   ROLES               AGE     VERSION
myuser-master-00   Ready    controlplane,etcd   6d19h   v1.14.6
myuser-worker-00   Ready    worker              6d19h   v1.14.6

kubectl delete clusterrolebinding myuser-cluster-admin
rolebinding.rbac.authorization.k8s.io/myuser-cluster-admin deleted

# Give admin access to this namespace
kubectl create rolebinding myuser-admin --namespace=myproject --clusterrole=admin --user=myuser
rolebinding.rbac.authorization.k8s.io/myuser-admin created

# Deploy an simple image
kubectl config use-context myuser
kubectl run --image bitnami/dokuwiki mydokuwiki --kubeconfig=myuser-config
kubectl get pods --kubeconfig=myuser-config
NAME                         READY   STATUS              RESTARTS   AGE
mydokuwiki-66c5dd668-w9m4g   0/1     ContainerCreating   0          12s

# Access to node level is restricted
kubectl get nodes  --kubeconfig=myuser-config
Error from server (Forbidden): nodes is forbidden: User "myuser" cannot list resource "nodes" in API group "" at the cluster scope

kubectl delete rolebinding myuser-admin --namespace=myproject
rolebinding.rbac.authorization.k8s.io/myuser-admin deleted

# Access to pods now is restricted
kubectl get pods --kubeconfig=myuser-config
Error from server (Forbidden): pods is forbidden: User "myuser" cannot list resource "pods" in API group "" in the namespace "myproject"

kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
    namespace: myproject
    name: deployment-manager
rules:
- apiGroups: [ "", "extensions", "apps" ]
  resources: [ "deployments", "replicasets", "pods" ]
  verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ]
  # You can also use ["*"]

kubectl create -f role-deployment-manager.yaml
role.rbac.authorization.k8s.io/deployment-manager created

kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: deployment-manager-binding
  namespace: myproject
subjects:
- kind: User
  name: myuser
  apiGroup: ""
roleRef:
  kind: Role
 name: deployment-manager
  apiGroup: ""

kubectl create -f rolebinding-deployment-manager.yaml

# You can access to this namespace again
kubectl get pods --kubeconfig=myuser-config
NAME                         READY   STATUS    RESTARTS   AGE
mydokuwiki-66c5dd668-w9m4g   1/1     Running   0          7m27s

# .. and run some operations in this namespace
kubectl delete pods mydokuwiki-66c5dd668-w9m4g --kubeconfig=myuser-config
pod "mydokuwiki-66c5dd668-w9m4g" deleted

kubectl delete rolebinding deployment-manager-binding
rolebinding.rbac.authorization.k8s.io/deployment-manager-binding deleted

Note

This works with LoadBalancer type, so therefore also works with the ingress-nginx-controller

--fixed-network
--fixed-subnet