From 882fb47936c3a96198fe50064ca63b27720a2536 Mon Sep 17 00:00:00 2001 From: pmandrik Date: Mon, 1 Nov 2021 15:55:20 +0100 Subject: [PATCH] add PVC cephfs --- other/cmsweb_k8_test/readme.txt | 83 ++++++++++++++----- other/cmsweb_k8_test/scripta/scripta.yaml | 2 +- .../scripta_cephfs/cephfs_claim.yaml | 28 +++++++ .../scripta_cephfs/scripta_cephfs.yaml | 41 +-------- .../scripta_cephfs/scripta_cephfs_back.yaml | 64 ++++++++++++++ other/sound_alarm_test.py | 2 +- 6 files changed, 159 insertions(+), 61 deletions(-) create mode 100644 other/cmsweb_k8_test/scripta_cephfs/cephfs_claim.yaml create mode 100644 other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs_back.yaml diff --git a/other/cmsweb_k8_test/readme.txt b/other/cmsweb_k8_test/readme.txt index e050edd..7bb2af0 100644 --- a/other/cmsweb_k8_test/readme.txt +++ b/other/cmsweb_k8_test/readme.txt @@ -1,12 +1,13 @@ ** Create personal VM:** https://clouddocs.web.cern.ch/tutorial/openstack_command_line.html -openstack server create --key-name lxplus --flavor m2.small --image "CC7 - x86_64 [2021-10-01]" vmpmandrik -openstack server show vmpmandrik -openstack volume create --size 20 vopmandrik +openstack server create --key-name lxplus --flavor m2.small --image "CC7 - x86_64 [2021-10-01]" vmname +openstack server show voname +openstack volume create --size 20 voname openstack volume list -openstack server add volume vmpmandrik vopmandrik -openstack console url show vmpmandrik -Login via : ssh root@vmpmandrik.cern.ch +openstack server add volume vmname voname +openstack console url show vmname +Login via : +ssh root@vmname.cern.ch **Run locally (assuming different consoles):** mkdir tmp @@ -48,12 +49,20 @@ test4 is a DQM dedicated testbed ( https://cms-http-group.docs.cern.ch/k8s_clust First test - this way docker container will create output files in the container /tmp: deploy-srv.sh scripta v1 test4 -Check the status: kubectl get pods -n default -To login to the pod: kubectl exec -it scripta-... sh -n default -To see logs of a pod: kubectl logs scripta-... -n default -To delete : kubectl delete pod scripta-6b4c867475-4xlzd - NOT WORKING -Check deployments: kubectl get deployments -n default -To delete : kubectl delete -n default deployment scripta +Check the status: +kubectl get pods -n default +To login to the pod: +kubectl exec -it scripta-... sh -n default +To see logs of a pod: +kubectl logs scripta-... -n default +To delete : +kubectl delete pod ${POD_NAME} - NOT WORKING +Check deployments: +kubectl get deployments -n default +To delete : +kubectl delete -n default deployment scripta +To get Pod logs: +kubectl describe pods ${POD_NAME} **K8 testbed deployment with eos **WIP** Following https://cms-http-group.docs.cern.ch/k8s_cluster/eos/ @@ -63,29 +72,61 @@ Check output /eos/project/c/cmsweb/www/dqm/k8test **K8 testbed deployment with cephfs** Create cephfs share volume: https://clouddocs.web.cern.ch/file_shares/quickstart.html Get local enviroment of DQM project following: https://clouddocs.web.cern.ch/using_openstack/environment_options.html -Set enviroment: -. CMS_DQM_DC_openrc.sh + +For DQM DC cluster: +. CMS_DQM_DC_openrc.sh - for DQM DC project Create a share: openstack share create --name myshare01 --share-type "Geneva CephFS Testing" CephFS 1 Check available share: manila list Add manila access-allow myshare01 cephx cmsweb-auth -Get osShareID : manila list myshare01 +Get osShareID : manila list Get osShareAccessID : manila access-list myshare01 osShareID & osShareAccessID Used in the k8 .yaml config Get mount path : manila share-export-location-list myshare01 +For CMSWEB K8 testbed clusters: +. CMS_WebtoolMig_openrc.sh +manila list +manila access-list pvc-c856be26-9de9-11e9-93c2-02163e01bcd6 + Create docker image as usual: docker build --build-arg CMSK8S=http://cmsweb-testbed.cern.ch -t pmandrik/scripta_cephfs:v1 scripta_cephfs docker run --rm -h `hostname -f` -v ~/tmp:/tmp -i -t pmandrik/scripta_cephfs:v1 docker push pmandrik/scripta_cephfs:v1 Deploy to k8: -deploy-srv.sh scripta_cephfs v1 test4 - -Mount at cluster: -ceph-fuse /cephfs/testbed/confdb-logs --id=cmsweb-auth --client-mountpoint=/volumes/_nogroup/9392e470-ef2c-4165-8caa-6063954e4e72 - - +kubectl apply -f cephfs_claim.yaml +kubectl apply -f scripta_cephfs.yaml + +Now not only the pods will be created but cephfs volumes and volums claims: +kubectl get pv -n default +kubectl get pvc -n default +To delete: +kubectl delete storageclass dqmpv --grace-period=0 --force +kubectl delete pv dqmpv --grace-period=0 --force +kubectl delete pvc dqmpv --grace-period=0 --force +To check: +kubectl describe storageclass dqmpv +kubectl describe pvc dqmpvc + +Mount at cluster from any VM (check Install ceph-fuse (for CC7) if ceph-fuse is not installed): +ceph-fuse /cephfs/testbed/confdb-logs --id=cmsweb-auth --client-mountpoint=/volumes/_nogroup/9bac97f9-3e4e-4627-9e1b-d2666d14b8fc + +** Install ceph-fuse (for CC7): ** +Follow https://manjusri.ucsc.edu/2017/09/25/ceph-fuse/ + +Create config /etc/ceph/ceph.conf with following content (for Geneva testing share): +[global] +admin socket = /var/run/ceph/\$cluster-\$name-\$pid.asok +client reconnect stale = true +debug client = 0/2 +fuse big writes = true +mon host = cephmond.cern.ch:6790 + +Create key config following +https://clouddocs.web.cern.ch/file_shares/manual_cephfs.html +and +https://github.com/dmwm/CMSKubernetes/blob/master/kubernetes/cmsweb/docs/storage.md diff --git a/other/cmsweb_k8_test/scripta/scripta.yaml b/other/cmsweb_k8_test/scripta/scripta.yaml index b4012e4..b8536c0 100644 --- a/other/cmsweb_k8_test/scripta/scripta.yaml +++ b/other/cmsweb_k8_test/scripta/scripta.yaml @@ -18,7 +18,7 @@ spec: spec: containers: - name: scripta - image: pmandrik/scripta #imagetag + image: pmandrik/scripta:v1 #imagetag resources: requests: memory: "100Mi" diff --git a/other/cmsweb_k8_test/scripta_cephfs/cephfs_claim.yaml b/other/cmsweb_k8_test/scripta_cephfs/cephfs_claim.yaml new file mode 100644 index 0000000..e7fdf3d --- /dev/null +++ b/other/cmsweb_k8_test/scripta_cephfs/cephfs_claim.yaml @@ -0,0 +1,28 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: dqmpv +provisioner: manila-provisioner +parameters: + type: "Geneva CephFS Testing" + zones: nova + osSecretName: os-trustee + osSecretNamespace: kube-system + protocol: CEPHFS + backend: csi-cephfs + csi-driver: cephfs.csi.ceph.com + osShareID: b8c03b1a-6606-4920-bacf-2a2990ceb091 # cec913f6-0016-4f59-9105-14d3abd4c607 + osShareAccessID: a08a9d8c-97e6-4b6a-b911-b439474c8f9c # 6169ece6-78f6-4be2-9450-4f71788bdfc0 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: dqmpvc +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Mi + storageClassName: dqmpvc + diff --git a/other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs.yaml b/other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs.yaml index c2cf35a..9aa9eab 100644 --- a/other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs.yaml +++ b/other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs.yaml @@ -1,39 +1,4 @@ --- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: existing-volume -spec: - accessModes: - - ReadWriteMany - capacity: - storage: 1G - csi: - driver: cephfs.csi.ceph.com - volumeHandle: cec913f6-0016-4f59-9105-14d3abd4c607 - nodeStageSecretRef: - name: os-trustee - namespace: kube-system - nodePublishSecretRef: - name: os-trustee - namespace: kube-system - volumeAttributes: - shareID: cec913f6-0016-4f59-9105-14d3abd4c607 - shareAccessID: 6169ece6-78f6-4be2-9450-4f71788bdfc0 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: existing-manila-pvc -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1G - storageClassName: "" - volumeName: existing-volume ---- kind: Deployment apiVersion: apps/v1 metadata: @@ -45,7 +10,7 @@ spec: selector: matchLabels: app: scripta - replicas: 2 + replicas: 1 template: # This is the pod template metadata: labels: @@ -53,7 +18,7 @@ spec: spec: containers: - name: scripta - image: pmandrik/scripta #imagetag + image: pmandrik/scripta_cephfs:v1 #imagetag resources: requests: memory: "100Mi" @@ -67,7 +32,7 @@ spec: volumes: - name: mypvc persistentVolumeClaim: - claimName: logs-cephfs-claim-preprod-default + claimName: dqmpvc readOnly: false diff --git a/other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs_back.yaml b/other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs_back.yaml new file mode 100644 index 0000000..5674cc6 --- /dev/null +++ b/other/cmsweb_k8_test/scripta_cephfs/scripta_cephfs_back.yaml @@ -0,0 +1,64 @@ +apiVersion: storage.k8s.io/v1beta1 # https://github.com/dmwm/CMSKubernetes/blob/master/kubernetes/cmsweb/storages/cephfs-storage-default.yaml +kind: StorageClass +metadata: + name: default-share + namespace: auth +provisioner: manila-provisioner +parameters: + type: "Geneva CephFS Testing" + zones: nova + osSecretName: os-trustee + osSecretNamespace: kube-system + protocol: CEPHFS + backend: csi-cephfs + csi-driver: cephfs.csi.ceph.com + osShareID: cec913f6-0016-4f59-9105-14d3abd4c607 + osShareAccessID: 6169ece6-78f6-4be2-9450-4f71788bdfc0 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: logs-cephfs-claim-preprod-default + namespace: auth +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1G + storageClassName: default-share +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: scripta + namespace: default + labels: + app: scripta +spec: + selector: + matchLabels: + app: scripta + replicas: 2 + template: # This is the pod template + metadata: + labels: + app: scripta + spec: + containers: + - name: scripta + image: pmandrik/scripta #imagetag + resources: + requests: + memory: "100Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "400m" + volumes: + - name: mypvc + persistentVolumeClaim: + claimName: logs-cephfs-claim-preprod-default + readOnly: false + + diff --git a/other/sound_alarm_test.py b/other/sound_alarm_test.py index d6e1404..5f1aa41 100644 --- a/other/sound_alarm_test.py +++ b/other/sound_alarm_test.py @@ -2,7 +2,7 @@ SOUNDSERVER = "daq-expert.cms" PORT = 50555 -MSGBODY = ('''%s Check plots in the DQM Error folder.') +MSGBODY = ('DQM test alarm.') from socket import socket, AF_INET, SOCK_STREAM, gethostname s = socket(AF_INET, SOCK_STREAM)